2024-11-24 16:40:25,376 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 16:40:25,392 main DEBUG Took 0.013509 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 16:40:25,393 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 16:40:25,393 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 16:40:25,394 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 16:40:25,396 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,406 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 16:40:25,429 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,431 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,432 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,434 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,435 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,437 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,438 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,439 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,441 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,442 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,444 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,445 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 16:40:25,446 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,446 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 16:40:25,448 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 16:40:25,453 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 16:40:25,458 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 16:40:25,458 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 16:40:25,460 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 16:40:25,461 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 16:40:25,472 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 16:40:25,475 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 16:40:25,477 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 16:40:25,478 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 16:40:25,478 main DEBUG createAppenders(={Console}) 2024-11-24 16:40:25,479 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-24 16:40:25,479 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 16:40:25,479 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-24 16:40:25,480 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 16:40:25,480 main DEBUG OutputStream closed 2024-11-24 16:40:25,481 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 16:40:25,481 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 16:40:25,482 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-24 16:40:25,589 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 16:40:25,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 16:40:25,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 16:40:25,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 16:40:25,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 16:40:25,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 16:40:25,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 16:40:25,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 16:40:25,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 16:40:25,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 16:40:25,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 16:40:25,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 16:40:25,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 16:40:25,601 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 16:40:25,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 16:40:25,602 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 16:40:25,603 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 16:40:25,604 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 16:40:25,607 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 16:40:25,608 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-24 16:40:25,608 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 16:40:25,609 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-24T16:40:25,945 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2 2024-11-24 16:40:25,948 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 16:40:25,949 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T16:40:25,961 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-24T16:40:25,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T16:40:26,007 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c, deleteOnExit=true 2024-11-24T16:40:26,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T16:40:26,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/test.cache.data in system properties and HBase conf 2024-11-24T16:40:26,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T16:40:26,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/hadoop.log.dir in system properties and HBase conf 2024-11-24T16:40:26,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T16:40:26,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T16:40:26,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T16:40:26,147 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T16:40:26,272 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T16:40:26,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T16:40:26,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T16:40:26,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T16:40:26,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T16:40:26,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T16:40:26,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T16:40:26,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T16:40:26,282 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T16:40:26,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T16:40:26,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/nfs.dump.dir in system properties and HBase conf 2024-11-24T16:40:26,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/java.io.tmpdir in system properties and HBase conf 2024-11-24T16:40:26,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T16:40:26,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T16:40:26,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T16:40:27,261 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T16:40:27,356 INFO [Time-limited test {}] log.Log(170): Logging initialized @2925ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T16:40:27,449 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T16:40:27,534 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T16:40:27,561 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T16:40:27,561 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T16:40:27,563 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T16:40:27,581 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T16:40:27,585 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/hadoop.log.dir/,AVAILABLE} 2024-11-24T16:40:27,586 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T16:40:27,818 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/java.io.tmpdir/jetty-localhost-37899-hadoop-hdfs-3_4_1-tests_jar-_-any-11078423472092210453/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T16:40:27,836 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37899} 2024-11-24T16:40:27,836 INFO [Time-limited test {}] server.Server(415): Started @3407ms 2024-11-24T16:40:28,273 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T16:40:28,280 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T16:40:28,281 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T16:40:28,281 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T16:40:28,281 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T16:40:28,282 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/hadoop.log.dir/,AVAILABLE} 2024-11-24T16:40:28,283 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T16:40:28,402 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/java.io.tmpdir/jetty-localhost-43401-hadoop-hdfs-3_4_1-tests_jar-_-any-3509381118856679825/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T16:40:28,403 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43401} 2024-11-24T16:40:28,403 INFO [Time-limited test {}] server.Server(415): Started @3974ms 2024-11-24T16:40:28,459 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T16:40:29,085 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/data/data2/current/BP-538122245-172.17.0.2-1732466426978/current, will proceed with Du for space computation calculation, 2024-11-24T16:40:29,092 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/data/data1/current/BP-538122245-172.17.0.2-1732466426978/current, will proceed with Du for space computation calculation, 2024-11-24T16:40:29,186 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T16:40:29,256 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc99c1ca0304157c with lease ID 0x22b3e3719407c827: Processing first storage report for DS-da0356d2-ec74-42c4-9242-a524cae31642 from datanode DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d7540ee3-cd6e-4ff0-9f26-dcf559838e09, infoPort=40267, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=608854028;c=1732466426978) 2024-11-24T16:40:29,259 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc99c1ca0304157c with lease ID 0x22b3e3719407c827: from storage DS-da0356d2-ec74-42c4-9242-a524cae31642 node DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d7540ee3-cd6e-4ff0-9f26-dcf559838e09, infoPort=40267, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=608854028;c=1732466426978), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-24T16:40:29,260 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfc99c1ca0304157c with lease ID 0x22b3e3719407c827: Processing first storage report for DS-dc84077e-fe3d-4c4e-86d7-969ad49dbe25 from datanode DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d7540ee3-cd6e-4ff0-9f26-dcf559838e09, infoPort=40267, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=608854028;c=1732466426978) 2024-11-24T16:40:29,261 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfc99c1ca0304157c with lease ID 0x22b3e3719407c827: from storage DS-dc84077e-fe3d-4c4e-86d7-969ad49dbe25 node DatanodeRegistration(127.0.0.1:34923, datanodeUuid=d7540ee3-cd6e-4ff0-9f26-dcf559838e09, infoPort=40267, infoSecurePort=0, ipcPort=42403, storageInfo=lv=-57;cid=testClusterID;nsid=608854028;c=1732466426978), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T16:40:29,334 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2 2024-11-24T16:40:29,441 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/zookeeper_0, clientPort=58212, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T16:40:29,454 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58212 2024-11-24T16:40:29,469 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T16:40:29,473 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T16:40:29,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741825_1001 (size=7) 2024-11-24T16:40:30,186 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 with version=8 2024-11-24T16:40:30,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/hbase-staging 2024-11-24T16:40:30,282 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T16:40:30,554 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:40:30,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:40:30,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:40:30,575 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:40:30,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:40:30,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:40:30,724 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T16:40:30,796 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T16:40:30,805 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T16:40:30,810 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:40:30,839 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1548 (auto-detected) 2024-11-24T16:40:30,840 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T16:40:30,861 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37979 2024-11-24T16:40:30,888 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37979 connecting to ZooKeeper ensemble=127.0.0.1:58212 2024-11-24T16:40:31,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379790x0, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:40:31,141 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37979-0x1014ad9b8770000 connected 2024-11-24T16:40:31,179 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T16:40:31,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T16:40:31,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:40:31,195 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446, hbase.cluster.distributed=false 2024-11-24T16:40:31,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:40:31,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37979 2024-11-24T16:40:31,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37979 2024-11-24T16:40:31,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37979 2024-11-24T16:40:31,240 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37979 2024-11-24T16:40:31,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37979 2024-11-24T16:40:31,401 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e335232f7491:0 server-side Connection retries=45 2024-11-24T16:40:31,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:40:31,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T16:40:31,405 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T16:40:31,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T16:40:31,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T16:40:31,409 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T16:40:31,412 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T16:40:31,413 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33607 2024-11-24T16:40:31,416 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33607 connecting to ZooKeeper ensemble=127.0.0.1:58212 2024-11-24T16:40:31,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T16:40:31,422 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T16:40:31,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336070x0, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T16:40:31,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:336070x0, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:40:31,436 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33607-0x1014ad9b8770001 connected 2024-11-24T16:40:31,442 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T16:40:31,455 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T16:40:31,458 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T16:40:31,464 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T16:40:31,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33607 2024-11-24T16:40:31,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33607 2024-11-24T16:40:31,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33607 2024-11-24T16:40:31,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33607 2024-11-24T16:40:31,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33607 2024-11-24T16:40:31,484 DEBUG [M:0;e335232f7491:37979 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e335232f7491:37979 2024-11-24T16:40:31,485 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e335232f7491,37979,1732466430336 2024-11-24T16:40:31,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:40:31,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:40:31,493 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e335232f7491,37979,1732466430336 2024-11-24T16:40:31,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:31,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T16:40:31,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:31,514 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T16:40:31,515 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e335232f7491,37979,1732466430336 from backup master directory 2024-11-24T16:40:31,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:40:31,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e335232f7491,37979,1732466430336 2024-11-24T16:40:31,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T16:40:31,519 WARN [master/e335232f7491:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:40:31,519 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e335232f7491,37979,1732466430336 2024-11-24T16:40:31,521 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T16:40:31,523 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T16:40:31,580 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/hbase.id] with ID: 83e3945b-9878-4aae-ad46-2020942a459f 2024-11-24T16:40:31,581 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/.tmp/hbase.id 2024-11-24T16:40:31,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741826_1002 (size=42) 2024-11-24T16:40:31,591 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/.tmp/hbase.id]:[hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/hbase.id] 2024-11-24T16:40:31,633 INFO [master/e335232f7491:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T16:40:31,639 INFO [master/e335232f7491:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T16:40:31,658 INFO [master/e335232f7491:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-24T16:40:31,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:31,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:31,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741827_1003 (size=196) 2024-11-24T16:40:31,695 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:40:31,697 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T16:40:31,714 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:31,718 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:40:31,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741828_1004 (size=1189) 2024-11-24T16:40:32,167 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store 2024-11-24T16:40:32,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741829_1005 (size=34) 2024-11-24T16:40:32,192 INFO [master/e335232f7491:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T16:40:32,195 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:40:32,197 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T16:40:32,197 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:40:32,197 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:40:32,199 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T16:40:32,199 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:40:32,200 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:40:32,202 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732466432197Disabling compacts and flushes for region at 1732466432197Disabling writes for close at 1732466432199 (+2 ms)Writing region close event to WAL at 1732466432200 (+1 ms)Closed at 1732466432200 2024-11-24T16:40:32,205 WARN [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/.initializing 2024-11-24T16:40:32,205 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/WALs/e335232f7491,37979,1732466430336 2024-11-24T16:40:32,217 INFO [master/e335232f7491:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:40:32,239 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e335232f7491%2C37979%2C1732466430336, suffix=, logDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/WALs/e335232f7491,37979,1732466430336, archiveDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/oldWALs, maxLogs=10 2024-11-24T16:40:32,274 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/WALs/e335232f7491,37979,1732466430336/e335232f7491%2C37979%2C1732466430336.1732466432245, exclude list is [], retry=0 2024-11-24T16:40:32,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34923,DS-da0356d2-ec74-42c4-9242-a524cae31642,DISK] 2024-11-24T16:40:32,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T16:40:32,343 INFO [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/WALs/e335232f7491,37979,1732466430336/e335232f7491%2C37979%2C1732466430336.1732466432245 2024-11-24T16:40:32,344 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-24T16:40:32,344 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:40:32,345 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:40:32,348 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,349 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T16:40:32,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:32,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:32,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T16:40:32,429 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:32,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:40:32,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T16:40:32,435 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:32,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:40:32,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T16:40:32,441 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:32,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:40:32,443 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,446 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,447 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,453 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,454 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,457 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T16:40:32,461 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T16:40:32,465 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:40:32,467 INFO [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70719294, jitterRate=0.053799599409103394}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T16:40:32,473 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732466432362Initializing all the Stores at 1732466432365 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466432365Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466432366 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466432367 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466432367Cleaning up temporary data from old regions at 1732466432454 (+87 ms)Region opened successfully at 1732466432473 (+19 ms) 2024-11-24T16:40:32,475 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T16:40:32,514 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d907c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:40:32,566 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T16:40:32,586 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T16:40:32,586 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T16:40:32,590 INFO [master/e335232f7491:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T16:40:32,592 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T16:40:32,598 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T16:40:32,598 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T16:40:32,628 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T16:40:32,641 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T16:40:32,644 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T16:40:32,647 INFO [master/e335232f7491:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T16:40:32,649 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T16:40:32,651 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T16:40:32,655 INFO [master/e335232f7491:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T16:40:32,661 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T16:40:32,662 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T16:40:32,665 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T16:40:32,668 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T16:40:32,687 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T16:40:32,689 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T16:40:32,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:40:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T16:40:32,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:32,701 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e335232f7491,37979,1732466430336, sessionid=0x1014ad9b8770000, setting cluster-up flag (Was=false) 2024-11-24T16:40:32,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:32,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:32,727 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T16:40:32,729 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e335232f7491,37979,1732466430336 2024-11-24T16:40:32,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:32,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:32,744 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T16:40:32,746 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e335232f7491,37979,1732466430336 2024-11-24T16:40:32,753 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T16:40:32,773 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(746): ClusterId : 83e3945b-9878-4aae-ad46-2020942a459f 2024-11-24T16:40:32,775 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T16:40:32,781 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T16:40:32,781 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T16:40:32,784 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T16:40:32,784 DEBUG [RS:0;e335232f7491:33607 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d17691, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e335232f7491/172.17.0.2:0 2024-11-24T16:40:32,800 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e335232f7491:33607 2024-11-24T16:40:32,803 INFO [RS:0;e335232f7491:33607 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T16:40:32,803 INFO [RS:0;e335232f7491:33607 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T16:40:32,803 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T16:40:32,806 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,37979,1732466430336 with port=33607, startcode=1732466431346 2024-11-24T16:40:32,818 DEBUG [RS:0;e335232f7491:33607 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T16:40:32,835 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T16:40:32,845 INFO [master/e335232f7491:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T16:40:32,852 INFO [master/e335232f7491:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T16:40:32,855 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44739, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T16:40:32,858 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e335232f7491,37979,1732466430336 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T16:40:32,863 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37979 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:32,866 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:40:32,866 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:40:32,867 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:40:32,867 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e335232f7491:0, corePoolSize=5, maxPoolSize=5 2024-11-24T16:40:32,867 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e335232f7491:0, corePoolSize=10, maxPoolSize=10 2024-11-24T16:40:32,867 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:32,868 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:40:32,868 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:32,873 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732466462873 2024-11-24T16:40:32,874 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T16:40:32,875 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T16:40:32,877 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:40:32,878 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T16:40:32,879 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T16:40:32,880 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T16:40:32,880 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T16:40:32,880 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T16:40:32,882 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:32,886 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:32,886 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T16:40:32,887 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T16:40:32,889 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T16:40:32,889 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T16:40:32,896 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T16:40:32,896 INFO [master/e335232f7491:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T16:40:32,897 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T16:40:32,898 WARN [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T16:40:32,898 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466432898,5,FailOnTimeoutGroup] 2024-11-24T16:40:32,899 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466432899,5,FailOnTimeoutGroup] 2024-11-24T16:40:32,899 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:32,899 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T16:40:32,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741831_1007 (size=1321) 2024-11-24T16:40:32,901 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:32,901 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:32,999 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(2659): reportForDuty to master=e335232f7491,37979,1732466430336 with port=33607, startcode=1732466431346 2024-11-24T16:40:33,004 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37979 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e335232f7491,33607,1732466431346 2024-11-24T16:40:33,007 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37979 {}] master.ServerManager(517): Registering regionserver=e335232f7491,33607,1732466431346 2024-11-24T16:40:33,015 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:40:33,015 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36643 2024-11-24T16:40:33,015 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T16:40:33,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:40:33,021 DEBUG [RS:0;e335232f7491:33607 {}] zookeeper.ZKUtil(111): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e335232f7491,33607,1732466431346 2024-11-24T16:40:33,022 WARN [RS:0;e335232f7491:33607 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T16:40:33,022 INFO [RS:0;e335232f7491:33607 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:40:33,022 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/WALs/e335232f7491,33607,1732466431346 2024-11-24T16:40:33,024 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e335232f7491,33607,1732466431346] 2024-11-24T16:40:33,054 INFO [RS:0;e335232f7491:33607 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T16:40:33,070 INFO [RS:0;e335232f7491:33607 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T16:40:33,075 INFO [RS:0;e335232f7491:33607 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T16:40:33,075 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,076 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T16:40:33,081 INFO [RS:0;e335232f7491:33607 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T16:40:33,082 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,083 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,083 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,083 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,083 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,083 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,083 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e335232f7491:0, corePoolSize=2, maxPoolSize=2 2024-11-24T16:40:33,084 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,084 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,084 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,084 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,084 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,084 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e335232f7491:0, corePoolSize=1, maxPoolSize=1 2024-11-24T16:40:33,084 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:40:33,085 DEBUG [RS:0;e335232f7491:33607 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0, corePoolSize=3, maxPoolSize=3 2024-11-24T16:40:33,085 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,086 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,086 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,086 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,086 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,086 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,33607,1732466431346-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:40:33,106 INFO [RS:0;e335232f7491:33607 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T16:40:33,108 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,33607,1732466431346-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,108 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,108 INFO [RS:0;e335232f7491:33607 {}] regionserver.Replication(171): e335232f7491,33607,1732466431346 started 2024-11-24T16:40:33,127 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:33,128 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1482): Serving as e335232f7491,33607,1732466431346, RpcServer on e335232f7491/172.17.0.2:33607, sessionid=0x1014ad9b8770001 2024-11-24T16:40:33,128 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T16:40:33,128 DEBUG [RS:0;e335232f7491:33607 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e335232f7491,33607,1732466431346 2024-11-24T16:40:33,129 DEBUG [RS:0;e335232f7491:33607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,33607,1732466431346' 2024-11-24T16:40:33,129 DEBUG [RS:0;e335232f7491:33607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T16:40:33,130 DEBUG [RS:0;e335232f7491:33607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T16:40:33,131 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T16:40:33,131 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T16:40:33,131 DEBUG [RS:0;e335232f7491:33607 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e335232f7491,33607,1732466431346 2024-11-24T16:40:33,131 DEBUG [RS:0;e335232f7491:33607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e335232f7491,33607,1732466431346' 2024-11-24T16:40:33,131 DEBUG [RS:0;e335232f7491:33607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T16:40:33,132 DEBUG [RS:0;e335232f7491:33607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T16:40:33,132 DEBUG [RS:0;e335232f7491:33607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T16:40:33,132 INFO [RS:0;e335232f7491:33607 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T16:40:33,133 INFO [RS:0;e335232f7491:33607 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T16:40:33,238 INFO [RS:0;e335232f7491:33607 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T16:40:33,241 INFO [RS:0;e335232f7491:33607 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e335232f7491%2C33607%2C1732466431346, suffix=, logDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/WALs/e335232f7491,33607,1732466431346, archiveDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/oldWALs, maxLogs=32 2024-11-24T16:40:33,260 DEBUG [RS:0;e335232f7491:33607 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/WALs/e335232f7491,33607,1732466431346/e335232f7491%2C33607%2C1732466431346.1732466433244, exclude list is [], retry=0 2024-11-24T16:40:33,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34923,DS-da0356d2-ec74-42c4-9242-a524cae31642,DISK] 2024-11-24T16:40:33,271 INFO [RS:0;e335232f7491:33607 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/WALs/e335232f7491,33607,1732466431346/e335232f7491%2C33607%2C1732466431346.1732466433244 2024-11-24T16:40:33,271 DEBUG [RS:0;e335232f7491:33607 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-24T16:40:33,303 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T16:40:33,303 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:40:33,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741833_1009 (size=32) 2024-11-24T16:40:33,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:40:33,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T16:40:33,723 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T16:40:33,723 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:33,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:33,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T16:40:33,727 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T16:40:33,727 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:33,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:33,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T16:40:33,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T16:40:33,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:33,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:33,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T16:40:33,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T16:40:33,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:33,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:33,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T16:40:33,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740 2024-11-24T16:40:33,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740 2024-11-24T16:40:33,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T16:40:33,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T16:40:33,740 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:40:33,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T16:40:33,746 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:40:33,746 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61772181, jitterRate=-0.07952277362346649}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:40:33,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732466433718Initializing all the Stores at 1732466433720 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466433720Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466433720Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466433720Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466433720Cleaning up temporary data from old regions at 1732466433739 (+19 ms)Region opened successfully at 1732466433748 (+9 ms) 2024-11-24T16:40:33,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T16:40:33,749 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T16:40:33,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T16:40:33,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T16:40:33,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T16:40:33,750 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T16:40:33,750 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732466433749Disabling compacts and flushes for region at 1732466433749Disabling writes for close at 1732466433749Writing region close event to WAL at 1732466433750 (+1 ms)Closed at 1732466433750 2024-11-24T16:40:33,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:40:33,754 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T16:40:33,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T16:40:33,768 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:40:33,771 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T16:40:33,923 DEBUG [e335232f7491:37979 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T16:40:33,936 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:40:33,942 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,33607,1732466431346, state=OPENING 2024-11-24T16:40:33,948 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T16:40:33,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:33,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:40:33,951 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:40:33,951 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:40:33,952 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T16:40:33,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:40:34,129 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T16:40:34,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33965, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T16:40:34,143 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T16:40:34,143 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T16:40:34,144 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T16:40:34,148 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e335232f7491%2C33607%2C1732466431346.meta, suffix=.meta, logDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/WALs/e335232f7491,33607,1732466431346, archiveDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/oldWALs, maxLogs=32 2024-11-24T16:40:34,173 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/WALs/e335232f7491,33607,1732466431346/e335232f7491%2C33607%2C1732466431346.meta.1732466434150.meta, exclude list is [], retry=0 2024-11-24T16:40:34,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34923,DS-da0356d2-ec74-42c4-9242-a524cae31642,DISK] 2024-11-24T16:40:34,180 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/WALs/e335232f7491,33607,1732466431346/e335232f7491%2C33607%2C1732466431346.meta.1732466434150.meta 2024-11-24T16:40:34,180 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40267:40267)] 2024-11-24T16:40:34,180 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:40:34,182 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T16:40:34,185 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T16:40:34,190 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T16:40:34,194 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T16:40:34,194 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:40:34,195 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T16:40:34,195 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T16:40:34,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T16:40:34,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T16:40:34,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:34,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:34,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T16:40:34,202 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T16:40:34,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:34,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:34,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T16:40:34,204 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T16:40:34,204 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:34,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:34,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T16:40:34,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T16:40:34,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:34,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T16:40:34,207 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T16:40:34,209 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740 2024-11-24T16:40:34,211 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740 2024-11-24T16:40:34,214 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T16:40:34,214 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T16:40:34,215 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:40:34,217 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T16:40:34,219 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61349057, jitterRate=-0.08582781255245209}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:40:34,219 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T16:40:34,221 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732466434195Writing region info on filesystem at 1732466434195Initializing all the Stores at 1732466434197 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466434197Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466434198 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466434198Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732466434198Cleaning up temporary data from old regions at 1732466434214 (+16 ms)Running coprocessor post-open hooks at 1732466434219 (+5 ms)Region opened successfully at 1732466434221 (+2 ms) 2024-11-24T16:40:34,228 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732466434120 2024-11-24T16:40:34,239 DEBUG [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T16:40:34,240 INFO [RS_OPEN_META-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T16:40:34,241 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:40:34,243 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e335232f7491,33607,1732466431346, state=OPEN 2024-11-24T16:40:34,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:40:34,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T16:40:34,252 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:40:34,252 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T16:40:34,252 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:34,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T16:40:34,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e335232f7491,33607,1732466431346 in 299 msec 2024-11-24T16:40:34,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T16:40:34,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 499 msec 2024-11-24T16:40:34,264 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T16:40:34,264 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T16:40:34,286 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:34,288 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:34,317 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:34,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:34,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5560 sec 2024-11-24T16:40:34,343 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732466434343, completionTime=-1 2024-11-24T16:40:34,346 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T16:40:34,346 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T16:40:34,373 INFO [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T16:40:34,373 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732466494373 2024-11-24T16:40:34,374 INFO [master/e335232f7491:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732466554374 2024-11-24T16:40:34,374 INFO [master/e335232f7491:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-24T16:40:34,376 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,37979,1732466430336-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:34,376 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,37979,1732466430336-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:34,376 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,37979,1732466430336-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:34,378 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e335232f7491:37979, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:34,378 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:34,380 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:34,386 DEBUG [master/e335232f7491:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T16:40:34,413 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.894sec 2024-11-24T16:40:34,414 INFO [master/e335232f7491:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T16:40:34,415 INFO [master/e335232f7491:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T16:40:34,416 INFO [master/e335232f7491:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T16:40:34,417 INFO [master/e335232f7491:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T16:40:34,417 INFO [master/e335232f7491:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T16:40:34,418 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,37979,1732466430336-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T16:40:34,418 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,37979,1732466430336-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T16:40:34,427 DEBUG [master/e335232f7491:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T16:40:34,428 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T16:40:34,428 INFO [master/e335232f7491:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e335232f7491,37979,1732466430336-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T16:40:34,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:34,485 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T16:40:34,485 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T16:40:34,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:34,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:34,499 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4640898b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:34,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:34,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:34,507 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:34,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:34,510 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:34,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:34,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:34,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e335232f7491,37979,1732466430336 2024-11-24T16:40:34,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=7113 2024-11-24T16:40:34,567 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:40:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,37979,1732466430336 2024-11-24T16:40:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b84ee7 2024-11-24T16:40:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:40:34,581 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:40:34,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:40:34,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:40:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T16:40:34,599 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:40:34,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-24T16:40:34,619 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:34,623 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:40:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:40:34,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741835_1011 (size=1061) 2024-11-24T16:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:40:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:40:35,042 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:40:35,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741836_1012 (size=53) 2024-11-24T16:40:35,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:40:35,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:40:35,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 950b8a656f36ed921dc512c3b64e83ba, disabling compactions & flushes 2024-11-24T16:40:35,452 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. after waiting 0 ms 2024-11-24T16:40:35,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,452 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,452 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 950b8a656f36ed921dc512c3b64e83ba: Waiting for close lock at 1732466435452Disabling compacts and flushes for region at 1732466435452Disabling writes for close at 1732466435452Writing region close event to WAL at 1732466435452Closed at 1732466435452 2024-11-24T16:40:35,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:40:35,460 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732466435455"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466435455"}]},"ts":"1732466435455"} 2024-11-24T16:40:35,464 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:40:35,465 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:40:35,468 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466435466"}]},"ts":"1732466435466"} 2024-11-24T16:40:35,472 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T16:40:35,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, ASSIGN}] 2024-11-24T16:40:35,476 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, ASSIGN 2024-11-24T16:40:35,477 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, ASSIGN; state=OFFLINE, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=false 2024-11-24T16:40:35,629 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=950b8a656f36ed921dc512c3b64e83ba, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:40:35,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, ASSIGN because future has completed 2024-11-24T16:40:35,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:40:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:40:35,794 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,794 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:40:35,795 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,795 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:40:35,795 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,795 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,798 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,801 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:40:35,801 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 950b8a656f36ed921dc512c3b64e83ba columnFamilyName A 2024-11-24T16:40:35,801 DEBUG [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:35,803 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.HStore(327): Store=950b8a656f36ed921dc512c3b64e83ba/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:40:35,803 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,805 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:40:35,805 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 950b8a656f36ed921dc512c3b64e83ba columnFamilyName B 2024-11-24T16:40:35,805 DEBUG [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:35,806 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.HStore(327): Store=950b8a656f36ed921dc512c3b64e83ba/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:40:35,806 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,808 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:40:35,808 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 950b8a656f36ed921dc512c3b64e83ba columnFamilyName C 2024-11-24T16:40:35,809 DEBUG [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:40:35,809 INFO [StoreOpener-950b8a656f36ed921dc512c3b64e83ba-1 {}] regionserver.HStore(327): Store=950b8a656f36ed921dc512c3b64e83ba/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:40:35,810 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,810 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,811 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,812 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,814 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,814 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,815 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:40:35,817 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,821 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:40:35,822 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 950b8a656f36ed921dc512c3b64e83ba; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64317300, jitterRate=-0.04159754514694214}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:40:35,822 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:35,823 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 950b8a656f36ed921dc512c3b64e83ba: Running coprocessor pre-open hook at 1732466435795Writing region info on filesystem at 1732466435795Initializing all the Stores at 1732466435797 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466435797Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466435797Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466435797Cleaning up temporary data from old regions at 1732466435814 (+17 ms)Running coprocessor post-open hooks at 1732466435822 (+8 ms)Region opened successfully at 1732466435823 (+1 ms) 2024-11-24T16:40:35,825 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., pid=6, masterSystemTime=1732466435788 2024-11-24T16:40:35,829 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,829 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:35,830 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=950b8a656f36ed921dc512c3b64e83ba, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:40:35,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:40:35,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T16:40:35,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 in 201 msec 2024-11-24T16:40:35,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T16:40:35,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, ASSIGN in 365 msec 2024-11-24T16:40:35,844 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:40:35,844 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466435844"}]},"ts":"1732466435844"} 2024-11-24T16:40:35,847 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T16:40:35,849 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:40:35,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2590 sec 2024-11-24T16:40:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T16:40:36,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:36,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,783 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63473470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302d7c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,788 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,794 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:40:36,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,37979,1732466430336 2024-11-24T16:40:36,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d32c91 2024-11-24T16:40:36,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:40:36,800 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:40:36,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,805 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4474c69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,808 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d05b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,811 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35afed03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,814 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4112bfce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,818 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cb0f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,821 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246c200c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,825 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5bc001, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,828 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0cd298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632c8962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,836 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6347268b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,841 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afb7308, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bf146b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,850 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fe38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,852 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60da3efe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,856 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9487f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,859 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff7d382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,860 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:40:36,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:40:36,862 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:40:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:40:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:40:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d9f899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:40:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:40:36,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:36,864 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:40:36,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d9ff7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:40:36,870 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,870 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,871 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,872 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:36,873 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,873 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,874 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,877 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,877 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-24T16:40:36,882 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T16:40:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,887 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:36,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,892 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:40:36,894 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,896 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:36,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:40:36,902 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:40:36,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:40:36,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:40:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:36,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:40:36,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:36,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:36,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:36,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T16:40:37,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T16:40:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:37,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466437065, max timestamp=1732466437093Num uniques 859; 2024-11-24T16:40:37,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:40:37,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/128c800074ed48be8f53c6705975f56b is 50, key is test_row_0/A:col10/1732466436955/Put/seqid=0 2024-11-24T16:40:37,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466437065, max timestamp=1732466437093Num uniques 859; 2024-11-24T16:40:37,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:40:37,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466437065, max timestamp=1732466437093Num uniques 859; 2024-11-24T16:40:37,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:40:37,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741837_1013 (size=12001) 2024-11-24T16:40:37,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/128c800074ed48be8f53c6705975f56b 2024-11-24T16:40:37,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T16:40:37,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466497202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466497202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466497208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466497208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466497214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/0d43a86dd067494281bc38b011e92495 is 50, key is test_row_0/B:col10/1732466436955/Put/seqid=0 2024-11-24T16:40:37,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T16:40:37,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741838_1014 (size=12001) 2024-11-24T16:40:37,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/0d43a86dd067494281bc38b011e92495 2024-11-24T16:40:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/a08247e63bb84c8da96bc0d593e21edf is 50, key is test_row_0/C:col10/1732466436955/Put/seqid=0 2024-11-24T16:40:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466497337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466497338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466497345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466497345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466497346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741839_1015 (size=12001) 2024-11-24T16:40:37,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/a08247e63bb84c8da96bc0d593e21edf 2024-11-24T16:40:37,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/128c800074ed48be8f53c6705975f56b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/128c800074ed48be8f53c6705975f56b 2024-11-24T16:40:37,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/128c800074ed48be8f53c6705975f56b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T16:40:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/0d43a86dd067494281bc38b011e92495 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0d43a86dd067494281bc38b011e92495 2024-11-24T16:40:37,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0d43a86dd067494281bc38b011e92495, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T16:40:37,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/a08247e63bb84c8da96bc0d593e21edf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a08247e63bb84c8da96bc0d593e21edf 2024-11-24T16:40:37,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T16:40:37,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:37,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a08247e63bb84c8da96bc0d593e21edf, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T16:40:37,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 950b8a656f36ed921dc512c3b64e83ba in 481ms, sequenceid=13, compaction requested=false 2024-11-24T16:40:37,456 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T16:40:37,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T16:40:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:37,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T16:40:37,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:37,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:37,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:37,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:37,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:37,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/c6ce6f29f62d47a6a1d0c9bf4b53de44 is 50, key is test_row_0/A:col10/1732466437208/Put/seqid=0 2024-11-24T16:40:37,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T16:40:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741840_1016 (size=14341) 2024-11-24T16:40:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/c6ce6f29f62d47a6a1d0c9bf4b53de44 2024-11-24T16:40:37,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466497596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466497598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466497604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/bca270ac0e99453d84dd0714b8746e49 is 50, key is test_row_0/B:col10/1732466437208/Put/seqid=0 2024-11-24T16:40:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466497607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466497610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741841_1017 (size=12001) 2024-11-24T16:40:37,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/bca270ac0e99453d84dd0714b8746e49 2024-11-24T16:40:37,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/091b53ea0f2b4acbbd8a0293ebe7a0ef is 50, key is test_row_0/C:col10/1732466437208/Put/seqid=0 2024-11-24T16:40:37,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741842_1018 (size=12001) 2024-11-24T16:40:37,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/091b53ea0f2b4acbbd8a0293ebe7a0ef 2024-11-24T16:40:37,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/c6ce6f29f62d47a6a1d0c9bf4b53de44 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/c6ce6f29f62d47a6a1d0c9bf4b53de44 2024-11-24T16:40:37,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466497715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466497728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/c6ce6f29f62d47a6a1d0c9bf4b53de44, entries=200, sequenceid=48, filesize=14.0 K 2024-11-24T16:40:37,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/bca270ac0e99453d84dd0714b8746e49 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/bca270ac0e99453d84dd0714b8746e49 2024-11-24T16:40:37,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466497745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T16:40:37,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466497746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:37,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466497746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:37,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:37,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/bca270ac0e99453d84dd0714b8746e49, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T16:40:37,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/091b53ea0f2b4acbbd8a0293ebe7a0ef as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/091b53ea0f2b4acbbd8a0293ebe7a0ef 2024-11-24T16:40:37,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/091b53ea0f2b4acbbd8a0293ebe7a0ef, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T16:40:37,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 950b8a656f36ed921dc512c3b64e83ba in 225ms, sequenceid=48, compaction requested=false 2024-11-24T16:40:37,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:37,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T16:40:37,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:37,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T16:40:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:37,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:37,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/45c3c5a501d84a558bcbd331459ebe79 is 50, key is test_row_0/A:col10/1732466437602/Put/seqid=0 2024-11-24T16:40:37,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:37,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741843_1019 (size=12001) 2024-11-24T16:40:37,950 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/45c3c5a501d84a558bcbd331459ebe79 2024-11-24T16:40:37,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/f20019b2e9e0494fbe1e7ea712f642cc is 50, key is test_row_0/B:col10/1732466437602/Put/seqid=0 2024-11-24T16:40:37,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741844_1020 (size=12001) 2024-11-24T16:40:37,997 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/f20019b2e9e0494fbe1e7ea712f642cc 2024-11-24T16:40:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466498004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466498008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T16:40:38,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466437999, max timestamp=1732466438013Num uniques 859; 2024-11-24T16:40:38,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466437999, max timestamp=1732466438013Num uniques 859; 2024-11-24T16:40:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466498013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:38,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/35d835583c514c9781abf2b58884724c is 50, key is test_row_0/C:col10/1732466437602/Put/seqid=0 2024-11-24T16:40:38,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466437999, max timestamp=1732466438013Num uniques 859; 2024-11-24T16:40:38,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:38,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741845_1021 (size=12001) 2024-11-24T16:40:38,066 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/35d835583c514c9781abf2b58884724c 2024-11-24T16:40:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/45c3c5a501d84a558bcbd331459ebe79 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/45c3c5a501d84a558bcbd331459ebe79 2024-11-24T16:40:38,097 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/45c3c5a501d84a558bcbd331459ebe79, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T16:40:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/f20019b2e9e0494fbe1e7ea712f642cc as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/f20019b2e9e0494fbe1e7ea712f642cc 2024-11-24T16:40:38,113 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/f20019b2e9e0494fbe1e7ea712f642cc, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T16:40:38,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/35d835583c514c9781abf2b58884724c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35d835583c514c9781abf2b58884724c 2024-11-24T16:40:38,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466498118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466498125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466498126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466498130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,137 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35d835583c514c9781abf2b58884724c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T16:40:38,139 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 950b8a656f36ed921dc512c3b64e83ba in 228ms, sequenceid=60, compaction requested=true 2024-11-24T16:40:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:38,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:38,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-24T16:40:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-24T16:40:38,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T16:40:38,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:38,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:38,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:38,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:38,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:38,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:38,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T16:40:38,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2500 sec 2024-11-24T16:40:38,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2890 sec 2024-11-24T16:40:38,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/a92c14ad15184497883c6b5090e73d1c is 50, key is test_row_0/A:col10/1732466438065/Put/seqid=0 2024-11-24T16:40:38,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741846_1022 (size=14341) 2024-11-24T16:40:38,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/a92c14ad15184497883c6b5090e73d1c 2024-11-24T16:40:38,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/ea07ff56b23245dc85308dd4bdc58e9e is 50, key is test_row_0/B:col10/1732466438065/Put/seqid=0 2024-11-24T16:40:38,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741847_1023 (size=12001) 2024-11-24T16:40:38,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466498234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466498235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466498245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466498344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466498345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466498345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466498444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466498454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466498574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/ea07ff56b23245dc85308dd4bdc58e9e 2024-11-24T16:40:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466498654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/35b630f3e7c44426b76253cf9ff6bb15 is 50, key is test_row_0/C:col10/1732466438065/Put/seqid=0 2024-11-24T16:40:38,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466498674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741848_1024 (size=12001) 2024-11-24T16:40:38,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/35b630f3e7c44426b76253cf9ff6bb15 2024-11-24T16:40:38,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/a92c14ad15184497883c6b5090e73d1c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a92c14ad15184497883c6b5090e73d1c 2024-11-24T16:40:38,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a92c14ad15184497883c6b5090e73d1c, entries=200, sequenceid=96, filesize=14.0 K 2024-11-24T16:40:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/ea07ff56b23245dc85308dd4bdc58e9e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ea07ff56b23245dc85308dd4bdc58e9e 2024-11-24T16:40:38,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466498754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466498764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:38,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:38,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:38,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ea07ff56b23245dc85308dd4bdc58e9e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T16:40:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/35b630f3e7c44426b76253cf9ff6bb15 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35b630f3e7c44426b76253cf9ff6bb15 2024-11-24T16:40:38,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35b630f3e7c44426b76253cf9ff6bb15, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T16:40:38,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 950b8a656f36ed921dc512c3b64e83ba in 645ms, sequenceid=96, compaction requested=true 2024-11-24T16:40:38,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:38,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:38,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:38,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:38,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:38,796 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:38,796 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:38,804 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:38,804 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:38,806 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:38,806 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:38,806 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:38,806 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:38,807 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/128c800074ed48be8f53c6705975f56b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/c6ce6f29f62d47a6a1d0c9bf4b53de44, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/45c3c5a501d84a558bcbd331459ebe79, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a92c14ad15184497883c6b5090e73d1c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=51.4 K 2024-11-24T16:40:38,807 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0d43a86dd067494281bc38b011e92495, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/bca270ac0e99453d84dd0714b8746e49, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/f20019b2e9e0494fbe1e7ea712f642cc, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ea07ff56b23245dc85308dd4bdc58e9e] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=46.9 K 2024-11-24T16:40:38,812 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d43a86dd067494281bc38b011e92495, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466436954 2024-11-24T16:40:38,813 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting bca270ac0e99453d84dd0714b8746e49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732466437200 2024-11-24T16:40:38,814 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f20019b2e9e0494fbe1e7ea712f642cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466437597 2024-11-24T16:40:38,815 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ea07ff56b23245dc85308dd4bdc58e9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466438065 2024-11-24T16:40:38,816 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128c800074ed48be8f53c6705975f56b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466436954 2024-11-24T16:40:38,820 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6ce6f29f62d47a6a1d0c9bf4b53de44, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732466437162 2024-11-24T16:40:38,822 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c3c5a501d84a558bcbd331459ebe79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466437597 2024-11-24T16:40:38,825 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a92c14ad15184497883c6b5090e73d1c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466438065 2024-11-24T16:40:38,867 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:38,869 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/cde3f8851544467389a5e63e11b67d4f is 50, key is test_row_0/B:col10/1732466438065/Put/seqid=0 2024-11-24T16:40:38,880 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#12 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:38,881 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/1e84e2019e574773a188a9e660542347 is 50, key is test_row_0/A:col10/1732466438065/Put/seqid=0 2024-11-24T16:40:38,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T16:40:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:38,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:38,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:38,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741849_1025 (size=12139) 2024-11-24T16:40:38,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/e5869c71d1b3434f9bc840df4d023a16 is 50, key is test_row_0/A:col10/1732466438208/Put/seqid=0 2024-11-24T16:40:38,932 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/cde3f8851544467389a5e63e11b67d4f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cde3f8851544467389a5e63e11b67d4f 2024-11-24T16:40:38,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741850_1026 (size=12139) 2024-11-24T16:40:38,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741851_1027 (size=14341) 2024-11-24T16:40:38,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/e5869c71d1b3434f9bc840df4d023a16 2024-11-24T16:40:38,964 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/1e84e2019e574773a188a9e660542347 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1e84e2019e574773a188a9e660542347 2024-11-24T16:40:38,972 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into cde3f8851544467389a5e63e11b67d4f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:38,972 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:38,975 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=12, startTime=1732466438795; duration=0sec 2024-11-24T16:40:38,975 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:38,975 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:38,975 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:38,986 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into 1e84e2019e574773a188a9e660542347(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:38,986 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:38,986 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:38,986 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=12, startTime=1732466438792; duration=0sec 2024-11-24T16:40:38,986 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:38,986 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:38,986 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:38,987 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:38,987 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a08247e63bb84c8da96bc0d593e21edf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/091b53ea0f2b4acbbd8a0293ebe7a0ef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35d835583c514c9781abf2b58884724c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35b630f3e7c44426b76253cf9ff6bb15] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=46.9 K 2024-11-24T16:40:38,988 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a08247e63bb84c8da96bc0d593e21edf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466436954 2024-11-24T16:40:38,989 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 091b53ea0f2b4acbbd8a0293ebe7a0ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732466437200 2024-11-24T16:40:38,991 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d835583c514c9781abf2b58884724c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466437597 2024-11-24T16:40:38,991 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b630f3e7c44426b76253cf9ff6bb15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466438065 2024-11-24T16:40:39,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fd09775f227143c1b1a00e6c52e620a4 is 50, key is test_row_0/B:col10/1732466438208/Put/seqid=0 2024-11-24T16:40:39,025 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#16 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:39,026 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d4a4ef61ba0d4b2ba654f464f3be2004 is 50, key is test_row_0/C:col10/1732466438065/Put/seqid=0 2024-11-24T16:40:39,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T16:40:39,035 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:39,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-24T16:40:39,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T16:40:39,041 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:39,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:39,047 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T16:40:39,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741852_1028 (size=12001) 2024-11-24T16:40:39,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fd09775f227143c1b1a00e6c52e620a4 2024-11-24T16:40:39,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741853_1029 (size=12139) 2024-11-24T16:40:39,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/f23efbc6c0fd4001b36f588f6e199460 is 50, key is test_row_0/C:col10/1732466438208/Put/seqid=0 2024-11-24T16:40:39,089 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d4a4ef61ba0d4b2ba654f464f3be2004 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d4a4ef61ba0d4b2ba654f464f3be2004 2024-11-24T16:40:39,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741854_1030 (size=12001) 2024-11-24T16:40:39,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/f23efbc6c0fd4001b36f588f6e199460 2024-11-24T16:40:39,123 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into d4a4ef61ba0d4b2ba654f464f3be2004(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:39,123 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:39,123 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=12, startTime=1732466438795; duration=0sec 2024-11-24T16:40:39,124 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:39,124 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:39,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466499125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/e5869c71d1b3434f9bc840df4d023a16 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/e5869c71d1b3434f9bc840df4d023a16 2024-11-24T16:40:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T16:40:39,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/e5869c71d1b3434f9bc840df4d023a16, entries=200, sequenceid=108, filesize=14.0 K 2024-11-24T16:40:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fd09775f227143c1b1a00e6c52e620a4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd09775f227143c1b1a00e6c52e620a4 2024-11-24T16:40:39,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd09775f227143c1b1a00e6c52e620a4, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T16:40:39,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/f23efbc6c0fd4001b36f588f6e199460 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f23efbc6c0fd4001b36f588f6e199460 2024-11-24T16:40:39,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466499164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f23efbc6c0fd4001b36f588f6e199460, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T16:40:39,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 950b8a656f36ed921dc512c3b64e83ba in 289ms, sequenceid=108, compaction requested=false 2024-11-24T16:40:39,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:39,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466439011, max timestamp=1732466439188Num uniques 859; 2024-11-24T16:40:39,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:39,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T16:40:39,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:39,197 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=82.74 KB heapSize=242.19 KB 2024-11-24T16:40:39,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466439011, max timestamp=1732466439188Num uniques 859; 2024-11-24T16:40:39,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:39,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:39,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:39,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:39,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/0a246fcd1baa40d981526f01b4f7899f is 50, key is test_row_0/A:col10/1732466439188/Put/seqid=0 2024-11-24T16:40:39,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741855_1031 (size=12101) 2024-11-24T16:40:39,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466439274, max timestamp=1732466439298Num uniques 859; 2024-11-24T16:40:39,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466439274, max timestamp=1732466439298Num uniques 859; 2024-11-24T16:40:39,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:39,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:39,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466439274, max timestamp=1732466439300Num uniques 859; 2024-11-24T16:40:39,309 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T16:40:39,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466499350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466499350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466499358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466499367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466499474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466499474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466499477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466499484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T16:40:39,666 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/0a246fcd1baa40d981526f01b4f7899f 2024-11-24T16:40:39,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1830371470c84243904944007f502217 is 50, key is test_row_0/B:col10/1732466439188/Put/seqid=0 2024-11-24T16:40:39,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466499684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466499685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466499694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466499694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:39,706 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:40:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:39,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:39,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741856_1032 (size=12101) 2024-11-24T16:40:39,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1830371470c84243904944007f502217 2024-11-24T16:40:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/da61c876c65641198734363ce871608d is 50, key is test_row_0/C:col10/1732466439188/Put/seqid=0 2024-11-24T16:40:39,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741857_1033 (size=12101) 2024-11-24T16:40:40,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466499995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466499995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466500015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466500015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T16:40:40,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466500185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,192 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:40,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,209 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/da61c876c65641198734363ce871608d 2024-11-24T16:40:40,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/0a246fcd1baa40d981526f01b4f7899f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/0a246fcd1baa40d981526f01b4f7899f 2024-11-24T16:40:40,243 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/0a246fcd1baa40d981526f01b4f7899f, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T16:40:40,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1830371470c84243904944007f502217 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1830371470c84243904944007f502217 2024-11-24T16:40:40,269 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1830371470c84243904944007f502217, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T16:40:40,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/da61c876c65641198734363ce871608d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/da61c876c65641198734363ce871608d 2024-11-24T16:40:40,286 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/da61c876c65641198734363ce871608d, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T16:40:40,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 950b8a656f36ed921dc512c3b64e83ba in 1091ms, sequenceid=132, compaction requested=true 2024-11-24T16:40:40,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-24T16:40:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-24T16:40:40,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T16:40:40,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2500 sec 2024-11-24T16:40:40,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2610 sec 2024-11-24T16:40:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:40,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T16:40:40,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:40,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:40:40,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:40,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:40:40,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:40,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:40:40,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/7d9e01a6f8984e90b1a9c7f4b19b5bce is 50, key is test_row_0/A:col10/1732466439357/Put/seqid=0 2024-11-24T16:40:40,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466500544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466500546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741858_1034 (size=16931) 2024-11-24T16:40:40,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/7d9e01a6f8984e90b1a9c7f4b19b5bce 2024-11-24T16:40:40,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466500556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466500555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/2355539ba80a42a984bb6c46e930ad4a is 50, key is test_row_0/B:col10/1732466439357/Put/seqid=0 2024-11-24T16:40:40,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741859_1035 (size=12151) 2024-11-24T16:40:40,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466500665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466500665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466500674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466500674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,790 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T16:40:40,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T16:40:40,794 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T16:40:40,794 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T16:40:40,794 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T16:40:40,794 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T16:40:40,796 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T16:40:40,796 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T16:40:40,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466500874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466500874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466500884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466500884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/2355539ba80a42a984bb6c46e930ad4a 2024-11-24T16:40:41,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/6516c8ea61124af48c57949c22cd04fa is 50, key is test_row_0/C:col10/1732466439357/Put/seqid=0 2024-11-24T16:40:41,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741860_1036 (size=12151) 2024-11-24T16:40:41,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/6516c8ea61124af48c57949c22cd04fa 2024-11-24T16:40:41,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/7d9e01a6f8984e90b1a9c7f4b19b5bce as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/7d9e01a6f8984e90b1a9c7f4b19b5bce 2024-11-24T16:40:41,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/7d9e01a6f8984e90b1a9c7f4b19b5bce, entries=250, sequenceid=170, filesize=16.5 K 2024-11-24T16:40:41,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/2355539ba80a42a984bb6c46e930ad4a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2355539ba80a42a984bb6c46e930ad4a 2024-11-24T16:40:41,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2355539ba80a42a984bb6c46e930ad4a, entries=150, sequenceid=170, filesize=11.9 K 2024-11-24T16:40:41,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/6516c8ea61124af48c57949c22cd04fa as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6516c8ea61124af48c57949c22cd04fa 2024-11-24T16:40:41,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6516c8ea61124af48c57949c22cd04fa, entries=150, sequenceid=170, filesize=11.9 K 2024-11-24T16:40:41,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=46.96 KB/48090 for 950b8a656f36ed921dc512c3b64e83ba in 587ms, sequenceid=170, compaction requested=true 2024-11-24T16:40:41,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:41,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:41,109 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:41,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:41,110 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:41,113 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:41,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:41,113 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:41,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:41,113 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:41,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:41,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:41,114 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1e84e2019e574773a188a9e660542347, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/e5869c71d1b3434f9bc840df4d023a16, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/0a246fcd1baa40d981526f01b4f7899f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/7d9e01a6f8984e90b1a9c7f4b19b5bce] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=54.2 K 2024-11-24T16:40:41,114 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:41,114 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:41,115 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:41,115 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cde3f8851544467389a5e63e11b67d4f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd09775f227143c1b1a00e6c52e620a4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1830371470c84243904944007f502217, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2355539ba80a42a984bb6c46e930ad4a] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=47.3 K 2024-11-24T16:40:41,117 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting cde3f8851544467389a5e63e11b67d4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466438065 2024-11-24T16:40:41,117 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e84e2019e574773a188a9e660542347, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466438065 2024-11-24T16:40:41,118 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fd09775f227143c1b1a00e6c52e620a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732466438208 2024-11-24T16:40:41,118 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5869c71d1b3434f9bc840df4d023a16, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732466438208 2024-11-24T16:40:41,119 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1830371470c84243904944007f502217, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732466439011 2024-11-24T16:40:41,120 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a246fcd1baa40d981526f01b4f7899f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732466439011 2024-11-24T16:40:41,120 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 2355539ba80a42a984bb6c46e930ad4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732466439349 2024-11-24T16:40:41,122 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d9e01a6f8984e90b1a9c7f4b19b5bce, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732466439345 2024-11-24T16:40:41,145 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:41,146 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/0206976fa0674ec48a8cdc0e22978839 is 50, key is test_row_0/B:col10/1732466439357/Put/seqid=0 2024-11-24T16:40:41,156 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:41,157 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4adc0ee67ed746a4ae28930abc51e562 is 50, key is test_row_0/A:col10/1732466439357/Put/seqid=0 2024-11-24T16:40:41,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741861_1037 (size=12425) 2024-11-24T16:40:41,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T16:40:41,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:41,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:41,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-24T16:40:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T16:40:41,189 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:41,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:40:41,191 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:41,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:41,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:41,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:41,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:41,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:41,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:41,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:41,207 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/0206976fa0674ec48a8cdc0e22978839 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0206976fa0674ec48a8cdc0e22978839 2024-11-24T16:40:41,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/ebf8d685cb9c4c9da791811a422ea64e is 50, key is test_row_0/A:col10/1732466441187/Put/seqid=0 2024-11-24T16:40:41,224 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into 0206976fa0674ec48a8cdc0e22978839(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:41,227 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:41,229 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=12, startTime=1732466441110; duration=0sec 2024-11-24T16:40:41,230 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:41,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741862_1038 (size=12425) 2024-11-24T16:40:41,230 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:41,230 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:41,232 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:41,233 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:41,233 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:41,233 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d4a4ef61ba0d4b2ba654f464f3be2004, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f23efbc6c0fd4001b36f588f6e199460, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/da61c876c65641198734363ce871608d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6516c8ea61124af48c57949c22cd04fa] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=47.3 K 2024-11-24T16:40:41,234 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a4ef61ba0d4b2ba654f464f3be2004, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466438065 2024-11-24T16:40:41,234 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f23efbc6c0fd4001b36f588f6e199460, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732466438208 2024-11-24T16:40:41,236 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting da61c876c65641198734363ce871608d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732466439011 2024-11-24T16:40:41,236 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6516c8ea61124af48c57949c22cd04fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732466439349 2024-11-24T16:40:41,257 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:41,258 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/90d89da8e557495b86f55d48c773d7d8 is 50, key is test_row_0/C:col10/1732466439357/Put/seqid=0 2024-11-24T16:40:41,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741863_1039 (size=16931) 2024-11-24T16:40:41,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466501250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/ebf8d685cb9c4c9da791811a422ea64e 2024-11-24T16:40:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466501252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466501255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466501261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fb6da2eefd3b4388999c7f133c042d1b is 50, key is test_row_0/B:col10/1732466441187/Put/seqid=0 2024-11-24T16:40:41,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741864_1040 (size=12425) 2024-11-24T16:40:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T16:40:41,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741865_1041 (size=12151) 2024-11-24T16:40:41,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fb6da2eefd3b4388999c7f133c042d1b 2024-11-24T16:40:41,309 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/90d89da8e557495b86f55d48c773d7d8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/90d89da8e557495b86f55d48c773d7d8 2024-11-24T16:40:41,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/509531cdbee0479dbb76661415d9e8ef is 50, key is test_row_0/C:col10/1732466441187/Put/seqid=0 2024-11-24T16:40:41,340 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into 90d89da8e557495b86f55d48c773d7d8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:41,340 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:41,340 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=12, startTime=1732466441113; duration=0sec 2024-11-24T16:40:41,340 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:41,340 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:41,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741866_1042 (size=12151) 2024-11-24T16:40:41,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/509531cdbee0479dbb76661415d9e8ef 2024-11-24T16:40:41,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T16:40:41,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:41,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:41,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:41,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:41,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:41,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/ebf8d685cb9c4c9da791811a422ea64e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/ebf8d685cb9c4c9da791811a422ea64e 2024-11-24T16:40:41,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/ebf8d685cb9c4c9da791811a422ea64e, entries=250, sequenceid=182, filesize=16.5 K 2024-11-24T16:40:41,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fb6da2eefd3b4388999c7f133c042d1b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fb6da2eefd3b4388999c7f133c042d1b 2024-11-24T16:40:41,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466501374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466501374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466501374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fb6da2eefd3b4388999c7f133c042d1b, entries=150, sequenceid=182, filesize=11.9 K 2024-11-24T16:40:41,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/509531cdbee0479dbb76661415d9e8ef as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/509531cdbee0479dbb76661415d9e8ef 2024-11-24T16:40:41,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466501384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/509531cdbee0479dbb76661415d9e8ef, entries=150, sequenceid=182, filesize=11.9 K 2024-11-24T16:40:41,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 950b8a656f36ed921dc512c3b64e83ba in 224ms, sequenceid=182, compaction requested=false 2024-11-24T16:40:41,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T16:40:41,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T16:40:41,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:41,508 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:40:41,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:41,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:41,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:41,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:41,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:41,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/870f8bddc7cd41aa922e0b90f7bb0dad is 50, key is test_row_0/A:col10/1732466441257/Put/seqid=0 2024-11-24T16:40:41,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741867_1043 (size=12151) 2024-11-24T16:40:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:41,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:41,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466501604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466501605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466501606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466501624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,634 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4adc0ee67ed746a4ae28930abc51e562 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4adc0ee67ed746a4ae28930abc51e562 2024-11-24T16:40:41,649 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into 4adc0ee67ed746a4ae28930abc51e562(size=12.1 K), total size for store is 28.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:41,650 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:41,650 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=12, startTime=1732466441108; duration=0sec 2024-11-24T16:40:41,650 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:41,650 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:41,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466501714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466501724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466501725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T16:40:41,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466501924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466501933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466501934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466501934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:41,939 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/870f8bddc7cd41aa922e0b90f7bb0dad 2024-11-24T16:40:41,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/54af325854f74dcbbb790e20e92f1800 is 50, key is test_row_0/B:col10/1732466441257/Put/seqid=0 2024-11-24T16:40:41,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741868_1044 (size=12151) 2024-11-24T16:40:41,970 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/54af325854f74dcbbb790e20e92f1800 2024-11-24T16:40:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/e792238ecba24d7bb860bd0653618b85 is 50, key is test_row_0/C:col10/1732466441257/Put/seqid=0 2024-11-24T16:40:41,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741869_1045 (size=12151) 2024-11-24T16:40:42,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466502214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,216 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466502233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466502244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466502244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T16:40:42,395 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/e792238ecba24d7bb860bd0653618b85 2024-11-24T16:40:42,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/870f8bddc7cd41aa922e0b90f7bb0dad as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/870f8bddc7cd41aa922e0b90f7bb0dad 2024-11-24T16:40:42,414 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/870f8bddc7cd41aa922e0b90f7bb0dad, entries=150, sequenceid=204, filesize=11.9 K 2024-11-24T16:40:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/54af325854f74dcbbb790e20e92f1800 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/54af325854f74dcbbb790e20e92f1800 2024-11-24T16:40:42,425 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/54af325854f74dcbbb790e20e92f1800, entries=150, sequenceid=204, filesize=11.9 K 2024-11-24T16:40:42,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/e792238ecba24d7bb860bd0653618b85 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/e792238ecba24d7bb860bd0653618b85 2024-11-24T16:40:42,437 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/e792238ecba24d7bb860bd0653618b85, entries=150, sequenceid=204, filesize=11.9 K 2024-11-24T16:40:42,438 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 950b8a656f36ed921dc512c3b64e83ba in 931ms, sequenceid=204, compaction requested=true 2024-11-24T16:40:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:42,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-24T16:40:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-24T16:40:42,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T16:40:42,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2510 sec 2024-11-24T16:40:42,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:40:42,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:42,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2620 sec 2024-11-24T16:40:42,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/5ee19f05eba941d9ba2137eff1e558f7 is 50, key is test_row_0/A:col10/1732466441602/Put/seqid=0 2024-11-24T16:40:42,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741870_1046 (size=14541) 2024-11-24T16:40:42,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/5ee19f05eba941d9ba2137eff1e558f7 2024-11-24T16:40:42,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/d69e451653b54a5a8c62c7c7cc070510 is 50, key is test_row_0/B:col10/1732466441602/Put/seqid=0 2024-11-24T16:40:42,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741871_1047 (size=12151) 2024-11-24T16:40:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466502577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466502693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466502744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466502754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466502754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/d69e451653b54a5a8c62c7c7cc070510 2024-11-24T16:40:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466502904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:42,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/a1e6b5368cb048b6a19043dee156ec97 is 50, key is test_row_0/C:col10/1732466441602/Put/seqid=0 2024-11-24T16:40:42,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741872_1048 (size=12151) 2024-11-24T16:40:42,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/a1e6b5368cb048b6a19043dee156ec97 2024-11-24T16:40:42,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/5ee19f05eba941d9ba2137eff1e558f7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5ee19f05eba941d9ba2137eff1e558f7 2024-11-24T16:40:42,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5ee19f05eba941d9ba2137eff1e558f7, entries=200, sequenceid=218, filesize=14.2 K 2024-11-24T16:40:42,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/d69e451653b54a5a8c62c7c7cc070510 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/d69e451653b54a5a8c62c7c7cc070510 2024-11-24T16:40:42,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/d69e451653b54a5a8c62c7c7cc070510, entries=150, sequenceid=218, filesize=11.9 K 2024-11-24T16:40:42,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/a1e6b5368cb048b6a19043dee156ec97 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a1e6b5368cb048b6a19043dee156ec97 2024-11-24T16:40:42,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a1e6b5368cb048b6a19043dee156ec97, entries=150, sequenceid=218, filesize=11.9 K 2024-11-24T16:40:42,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 950b8a656f36ed921dc512c3b64e83ba in 512ms, sequenceid=218, compaction requested=true 2024-11-24T16:40:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:42,960 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:42,960 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:42,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:42,963 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:42,963 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:42,963 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:42,963 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0206976fa0674ec48a8cdc0e22978839, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fb6da2eefd3b4388999c7f133c042d1b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/54af325854f74dcbbb790e20e92f1800, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/d69e451653b54a5a8c62c7c7cc070510] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=47.7 K 2024-11-24T16:40:42,964 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0206976fa0674ec48a8cdc0e22978839, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732466439349 2024-11-24T16:40:42,964 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fb6da2eefd3b4388999c7f133c042d1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732466440543 2024-11-24T16:40:42,964 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:42,965 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:42,965 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:42,965 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4adc0ee67ed746a4ae28930abc51e562, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/ebf8d685cb9c4c9da791811a422ea64e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/870f8bddc7cd41aa922e0b90f7bb0dad, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5ee19f05eba941d9ba2137eff1e558f7] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=54.7 K 2024-11-24T16:40:42,965 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 54af325854f74dcbbb790e20e92f1800, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732466441244 2024-11-24T16:40:42,965 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4adc0ee67ed746a4ae28930abc51e562, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732466439349 2024-11-24T16:40:42,966 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d69e451653b54a5a8c62c7c7cc070510, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732466441602 2024-11-24T16:40:42,966 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebf8d685cb9c4c9da791811a422ea64e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732466440543 2024-11-24T16:40:42,967 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 870f8bddc7cd41aa922e0b90f7bb0dad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732466441244 2024-11-24T16:40:42,967 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee19f05eba941d9ba2137eff1e558f7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732466441602 2024-11-24T16:40:42,981 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:42,981 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/2f5446ee6f014993b2d515a6b5896244 is 50, key is test_row_0/B:col10/1732466441602/Put/seqid=0 2024-11-24T16:40:42,982 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:42,983 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/42b41c7f91fe459dbbdf0dc048ed4a11 is 50, key is test_row_0/A:col10/1732466441602/Put/seqid=0 2024-11-24T16:40:42,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741873_1049 (size=12561) 2024-11-24T16:40:42,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741874_1050 (size=12561) 2024-11-24T16:40:43,009 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/42b41c7f91fe459dbbdf0dc048ed4a11 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/42b41c7f91fe459dbbdf0dc048ed4a11 2024-11-24T16:40:43,018 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into 42b41c7f91fe459dbbdf0dc048ed4a11(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:43,018 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:43,018 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=12, startTime=1732466442960; duration=0sec 2024-11-24T16:40:43,019 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:43,019 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:43,020 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:43,023 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:43,023 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:43,023 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:43,024 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/90d89da8e557495b86f55d48c773d7d8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/509531cdbee0479dbb76661415d9e8ef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/e792238ecba24d7bb860bd0653618b85, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a1e6b5368cb048b6a19043dee156ec97] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=47.7 K 2024-11-24T16:40:43,024 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d89da8e557495b86f55d48c773d7d8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732466439349 2024-11-24T16:40:43,025 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 509531cdbee0479dbb76661415d9e8ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732466440543 2024-11-24T16:40:43,025 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e792238ecba24d7bb860bd0653618b85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732466441244 2024-11-24T16:40:43,026 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e6b5368cb048b6a19043dee156ec97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732466441602 2024-11-24T16:40:43,049 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#38 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:43,050 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/db3f581d2d0940e8af31e80188d479dc is 50, key is test_row_0/C:col10/1732466441602/Put/seqid=0 2024-11-24T16:40:43,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741875_1051 (size=12561) 2024-11-24T16:40:43,087 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/db3f581d2d0940e8af31e80188d479dc as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/db3f581d2d0940e8af31e80188d479dc 2024-11-24T16:40:43,097 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into db3f581d2d0940e8af31e80188d479dc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:43,098 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:43,098 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=12, startTime=1732466442960; duration=0sec 2024-11-24T16:40:43,098 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:43,098 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:43,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:40:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:43,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:43,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/6cc58532608542539a9f4249a9cb9d23 is 50, key is test_row_0/A:col10/1732466443217/Put/seqid=0 2024-11-24T16:40:43,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741876_1052 (size=12151) 2024-11-24T16:40:43,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/6cc58532608542539a9f4249a9cb9d23 2024-11-24T16:40:43,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/180f6604e3de4964b152adb0d56f6d4b is 50, key is test_row_0/B:col10/1732466443217/Put/seqid=0 2024-11-24T16:40:43,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741877_1053 (size=12151) 2024-11-24T16:40:43,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/180f6604e3de4964b152adb0d56f6d4b 2024-11-24T16:40:43,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466503287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:43,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7 is 50, key is test_row_0/C:col10/1732466443217/Put/seqid=0 2024-11-24T16:40:43,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741878_1054 (size=12151) 2024-11-24T16:40:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T16:40:43,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:43,335 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:43,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-24T16:40:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T16:40:43,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:43,338 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:43,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:43,402 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/2f5446ee6f014993b2d515a6b5896244 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2f5446ee6f014993b2d515a6b5896244 2024-11-24T16:40:43,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:43,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466503404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:43,419 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into 2f5446ee6f014993b2d515a6b5896244(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:43,419 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:43,419 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=12, startTime=1732466442960; duration=0sec 2024-11-24T16:40:43,422 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:43,422 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T16:40:43,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T16:40:43,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:43,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:43,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:43,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:43,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466503613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:43,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T16:40:43,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:43,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:43,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:43,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T16:40:43,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7 2024-11-24T16:40:43,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/6cc58532608542539a9f4249a9cb9d23 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6cc58532608542539a9f4249a9cb9d23 2024-11-24T16:40:43,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6cc58532608542539a9f4249a9cb9d23, entries=150, sequenceid=240, filesize=11.9 K 2024-11-24T16:40:43,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/180f6604e3de4964b152adb0d56f6d4b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/180f6604e3de4964b152adb0d56f6d4b 2024-11-24T16:40:43,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/180f6604e3de4964b152adb0d56f6d4b, entries=150, sequenceid=240, filesize=11.9 K 2024-11-24T16:40:43,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7 2024-11-24T16:40:43,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:43,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466503763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:43,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:43,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466503764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:43,765 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466503764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:43,766 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:43,767 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:43,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7, entries=150, sequenceid=240, filesize=11.9 K 2024-11-24T16:40:43,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 950b8a656f36ed921dc512c3b64e83ba in 555ms, sequenceid=240, compaction requested=false 2024-11-24T16:40:43,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:43,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T16:40:43,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:43,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T16:40:43,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:43,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:43,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/774e0c7984e54e3d80e51a56353fca5f is 50, key is test_row_0/A:col10/1732466443264/Put/seqid=0 2024-11-24T16:40:43,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741879_1055 (size=12151) 2024-11-24T16:40:43,826 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/774e0c7984e54e3d80e51a56353fca5f 2024-11-24T16:40:43,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fabfda338f4b4eb889d585a2de7754cb is 50, key is test_row_0/B:col10/1732466443264/Put/seqid=0 2024-11-24T16:40:43,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741880_1056 (size=12151) 2024-11-24T16:40:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:43,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T16:40:44,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466504009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,012 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:44,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466504114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:44,248 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fabfda338f4b4eb889d585a2de7754cb 2024-11-24T16:40:44,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d10c557dcd74482f8a9a8866f0952e04 is 50, key is test_row_0/C:col10/1732466443264/Put/seqid=0 2024-11-24T16:40:44,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741881_1057 (size=12151) 2024-11-24T16:40:44,269 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d10c557dcd74482f8a9a8866f0952e04 2024-11-24T16:40:44,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/774e0c7984e54e3d80e51a56353fca5f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/774e0c7984e54e3d80e51a56353fca5f 2024-11-24T16:40:44,283 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/774e0c7984e54e3d80e51a56353fca5f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-24T16:40:44,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fabfda338f4b4eb889d585a2de7754cb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fabfda338f4b4eb889d585a2de7754cb 2024-11-24T16:40:44,296 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fabfda338f4b4eb889d585a2de7754cb, entries=150, sequenceid=253, filesize=11.9 K 2024-11-24T16:40:44,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d10c557dcd74482f8a9a8866f0952e04 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d10c557dcd74482f8a9a8866f0952e04 2024-11-24T16:40:44,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d10c557dcd74482f8a9a8866f0952e04, entries=150, sequenceid=253, filesize=11.9 K 2024-11-24T16:40:44,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 950b8a656f36ed921dc512c3b64e83ba in 503ms, sequenceid=253, compaction requested=true 2024-11-24T16:40:44,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-24T16:40:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-24T16:40:44,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T16:40:44,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 971 msec 2024-11-24T16:40:44,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 979 msec 2024-11-24T16:40:44,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T16:40:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:44,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:44,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/26988a3c72004e09bfe09fe8c11156f5 is 50, key is test_row_0/A:col10/1732466444001/Put/seqid=0 2024-11-24T16:40:44,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741882_1058 (size=14741) 2024-11-24T16:40:44,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/26988a3c72004e09bfe09fe8c11156f5 2024-11-24T16:40:44,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466504396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:44,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b243e540f9e942df86f9494a48479d4c is 50, key is test_row_0/B:col10/1732466444001/Put/seqid=0 2024-11-24T16:40:44,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741883_1059 (size=12301) 2024-11-24T16:40:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T16:40:44,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:44,476 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-24T16:40:44,479 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T16:40:44,480 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:44,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466504503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T16:40:44,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T16:40:44,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:44,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:44,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:44,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466504713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:44,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:44,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:44,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T16:40:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:44,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T16:40:44,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b243e540f9e942df86f9494a48479d4c 2024-11-24T16:40:44,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3a80e7446d0b446095043dee021c01ef is 50, key is test_row_0/C:col10/1732466444001/Put/seqid=0 2024-11-24T16:40:44,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741884_1060 (size=12301) 2024-11-24T16:40:44,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3a80e7446d0b446095043dee021c01ef 2024-11-24T16:40:44,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/26988a3c72004e09bfe09fe8c11156f5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/26988a3c72004e09bfe09fe8c11156f5 2024-11-24T16:40:44,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/26988a3c72004e09bfe09fe8c11156f5, entries=200, sequenceid=274, filesize=14.4 K 2024-11-24T16:40:44,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b243e540f9e942df86f9494a48479d4c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b243e540f9e942df86f9494a48479d4c 2024-11-24T16:40:44,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b243e540f9e942df86f9494a48479d4c, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T16:40:44,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3a80e7446d0b446095043dee021c01ef as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3a80e7446d0b446095043dee021c01ef 2024-11-24T16:40:44,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3a80e7446d0b446095043dee021c01ef, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T16:40:44,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 950b8a656f36ed921dc512c3b64e83ba in 548ms, sequenceid=274, compaction requested=true 2024-11-24T16:40:44,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:44,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:44,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:44,881 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:44,881 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:44,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:44,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:44,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:44,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:44,883 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49164 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:44,883 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:44,883 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,883 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2f5446ee6f014993b2d515a6b5896244, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/180f6604e3de4964b152adb0d56f6d4b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fabfda338f4b4eb889d585a2de7754cb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b243e540f9e942df86f9494a48479d4c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=48.0 K 2024-11-24T16:40:44,884 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:44,884 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:44,884 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,884 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5446ee6f014993b2d515a6b5896244, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732466441602 2024-11-24T16:40:44,884 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/42b41c7f91fe459dbbdf0dc048ed4a11, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6cc58532608542539a9f4249a9cb9d23, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/774e0c7984e54e3d80e51a56353fca5f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/26988a3c72004e09bfe09fe8c11156f5] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=50.4 K 2024-11-24T16:40:44,885 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b41c7f91fe459dbbdf0dc048ed4a11, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732466441602 2024-11-24T16:40:44,885 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 180f6604e3de4964b152adb0d56f6d4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732466442568 2024-11-24T16:40:44,885 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cc58532608542539a9f4249a9cb9d23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732466442568 2024-11-24T16:40:44,885 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fabfda338f4b4eb889d585a2de7754cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732466443264 2024-11-24T16:40:44,886 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774e0c7984e54e3d80e51a56353fca5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732466443264 2024-11-24T16:40:44,886 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting b243e540f9e942df86f9494a48479d4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732466444001 2024-11-24T16:40:44,887 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26988a3c72004e09bfe09fe8c11156f5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732466443988 2024-11-24T16:40:44,908 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:44,909 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/751864c62b634adea1908f9d18308de3 is 50, key is test_row_0/B:col10/1732466444001/Put/seqid=0 2024-11-24T16:40:44,912 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:44,913 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/9020fc6452004d20812baf8df60c5928 is 50, key is test_row_0/A:col10/1732466444001/Put/seqid=0 2024-11-24T16:40:44,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741885_1061 (size=12847) 2024-11-24T16:40:44,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741886_1062 (size=12847) 2024-11-24T16:40:44,932 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/9020fc6452004d20812baf8df60c5928 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/9020fc6452004d20812baf8df60c5928 2024-11-24T16:40:44,941 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into 9020fc6452004d20812baf8df60c5928(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:44,941 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:44,941 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=12, startTime=1732466444881; duration=0sec 2024-11-24T16:40:44,941 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:44,941 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:44,941 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:44,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T16:40:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,944 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:44,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:44,945 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49164 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:44,945 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:44,945 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:44,945 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/db3f581d2d0940e8af31e80188d479dc, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d10c557dcd74482f8a9a8866f0952e04, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3a80e7446d0b446095043dee021c01ef] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=48.0 K 2024-11-24T16:40:44,945 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting db3f581d2d0940e8af31e80188d479dc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732466441602 2024-11-24T16:40:44,946 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9b0d25f4f84a8eb26c3163e6b2f5f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1732466442568 2024-11-24T16:40:44,946 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting d10c557dcd74482f8a9a8866f0952e04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732466443264 2024-11-24T16:40:44,947 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a80e7446d0b446095043dee021c01ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732466444001 2024-11-24T16:40:44,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/1cd51bbcd7be459dabdbc5fd5853c1fd is 50, key is test_row_0/A:col10/1732466444375/Put/seqid=0 2024-11-24T16:40:44,973 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:44,973 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/1c3be3ab8cd046e59ddbe359c10dfc66 is 50, key is test_row_0/C:col10/1732466444001/Put/seqid=0 2024-11-24T16:40:44,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741887_1063 (size=12301) 2024-11-24T16:40:44,976 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/1cd51bbcd7be459dabdbc5fd5853c1fd 2024-11-24T16:40:44,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/a36c1c4b2caf4599ac60f2a255e83675 is 50, key is test_row_0/B:col10/1732466444375/Put/seqid=0 2024-11-24T16:40:44,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741888_1064 (size=12847) 2024-11-24T16:40:45,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741889_1065 (size=12301) 2024-11-24T16:40:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:45,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T16:40:45,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466445071, max timestamp=1732466445125Num uniques 859; 2024-11-24T16:40:45,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:40:45,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466445071, max timestamp=1732466445125Num uniques 859; 2024-11-24T16:40:45,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:40:45,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466445071, max timestamp=1732466445125Num uniques 859; 2024-11-24T16:40:45,133 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:45,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466505242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,322 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/751864c62b634adea1908f9d18308de3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/751864c62b634adea1908f9d18308de3 2024-11-24T16:40:45,329 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into 751864c62b634adea1908f9d18308de3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:45,329 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:45,329 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=12, startTime=1732466444881; duration=0sec 2024-11-24T16:40:45,329 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:45,329 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:45,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466505354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,402 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/1c3be3ab8cd046e59ddbe359c10dfc66 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1c3be3ab8cd046e59ddbe359c10dfc66 2024-11-24T16:40:45,403 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/a36c1c4b2caf4599ac60f2a255e83675 2024-11-24T16:40:45,411 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into 1c3be3ab8cd046e59ddbe359c10dfc66(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:45,411 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:45,411 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=12, startTime=1732466444881; duration=0sec 2024-11-24T16:40:45,411 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:45,411 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:45,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/2d3bc21f6e774c40b39900047959b267 is 50, key is test_row_0/C:col10/1732466444375/Put/seqid=0 2024-11-24T16:40:45,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741890_1066 (size=12301) 2024-11-24T16:40:45,422 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/2d3bc21f6e774c40b39900047959b267 2024-11-24T16:40:45,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/1cd51bbcd7be459dabdbc5fd5853c1fd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1cd51bbcd7be459dabdbc5fd5853c1fd 2024-11-24T16:40:45,437 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1cd51bbcd7be459dabdbc5fd5853c1fd, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T16:40:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/a36c1c4b2caf4599ac60f2a255e83675 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/a36c1c4b2caf4599ac60f2a255e83675 2024-11-24T16:40:45,447 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/a36c1c4b2caf4599ac60f2a255e83675, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T16:40:45,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/2d3bc21f6e774c40b39900047959b267 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/2d3bc21f6e774c40b39900047959b267 2024-11-24T16:40:45,456 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/2d3bc21f6e774c40b39900047959b267, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T16:40:45,457 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 950b8a656f36ed921dc512c3b64e83ba in 513ms, sequenceid=286, compaction requested=false 2024-11-24T16:40:45,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:45,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:45,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-24T16:40:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-24T16:40:45,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T16:40:45,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 980 msec 2024-11-24T16:40:45,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 988 msec 2024-11-24T16:40:45,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:45,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-11-24T16:40:45,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:45,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:45,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:45,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:45,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:45,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:45,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/5c2083e2f8ff4b448e0cca60450bf28a is 50, key is test_row_0/A:col10/1732466445232/Put/seqid=0 2024-11-24T16:40:45,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741891_1067 (size=14741) 2024-11-24T16:40:45,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/5c2083e2f8ff4b448e0cca60450bf28a 2024-11-24T16:40:45,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/cc8c6b296f6e40e2925a01f3247ae316 is 50, key is test_row_0/B:col10/1732466445232/Put/seqid=0 2024-11-24T16:40:45,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466505600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741892_1068 (size=12301) 2024-11-24T16:40:45,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/cc8c6b296f6e40e2925a01f3247ae316 2024-11-24T16:40:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T16:40:45,614 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:45,615 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-24T16:40:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T16:40:45,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:45,619 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:45,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:45,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3e4e3699ebc64ab68b8c25291c67b87e is 50, key is test_row_0/C:col10/1732466445232/Put/seqid=0 2024-11-24T16:40:45,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741893_1069 (size=12301) 2024-11-24T16:40:45,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466505714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T16:40:45,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T16:40:45,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:45,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:45,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:45,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:45,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466505773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,776 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466505784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,790 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466505794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,799 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:45,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466505924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:45,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T16:40:45,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:45,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T16:40:46,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3e4e3699ebc64ab68b8c25291c67b87e 2024-11-24T16:40:46,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/5c2083e2f8ff4b448e0cca60450bf28a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5c2083e2f8ff4b448e0cca60450bf28a 2024-11-24T16:40:46,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5c2083e2f8ff4b448e0cca60450bf28a, entries=200, sequenceid=324, filesize=14.4 K 2024-11-24T16:40:46,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/cc8c6b296f6e40e2925a01f3247ae316 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc8c6b296f6e40e2925a01f3247ae316 2024-11-24T16:40:46,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc8c6b296f6e40e2925a01f3247ae316, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T16:40:46,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3e4e3699ebc64ab68b8c25291c67b87e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3e4e3699ebc64ab68b8c25291c67b87e 2024-11-24T16:40:46,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3e4e3699ebc64ab68b8c25291c67b87e, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T16:40:46,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=46.96 KB/48090 for 950b8a656f36ed921dc512c3b64e83ba in 491ms, sequenceid=324, compaction requested=true 2024-11-24T16:40:46,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:46,058 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:46,058 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:46,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:46,059 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:46,059 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:46,059 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:46,059 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:46,059 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:46,059 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/9020fc6452004d20812baf8df60c5928, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1cd51bbcd7be459dabdbc5fd5853c1fd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5c2083e2f8ff4b448e0cca60450bf28a] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=39.0 K 2024-11-24T16:40:46,059 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:46,060 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/751864c62b634adea1908f9d18308de3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/a36c1c4b2caf4599ac60f2a255e83675, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc8c6b296f6e40e2925a01f3247ae316] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=36.6 K 2024-11-24T16:40:46,060 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9020fc6452004d20812baf8df60c5928, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732466444001 2024-11-24T16:40:46,060 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 751864c62b634adea1908f9d18308de3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732466444001 2024-11-24T16:40:46,060 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a36c1c4b2caf4599ac60f2a255e83675, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732466444361 2024-11-24T16:40:46,061 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd51bbcd7be459dabdbc5fd5853c1fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732466444361 2024-11-24T16:40:46,061 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8c6b296f6e40e2925a01f3247ae316, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466445232 2024-11-24T16:40:46,061 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c2083e2f8ff4b448e0cca60450bf28a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466445195 2024-11-24T16:40:46,082 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:46,082 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#57 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:46,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T16:40:46,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:46,083 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:46,083 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/eba52c95704f41119c2264e5d4dfb78e is 50, key is test_row_0/A:col10/1732466445232/Put/seqid=0 2024-11-24T16:40:46,083 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/ec00ce162b7e4cfc8ce81954b97d0de5 is 50, key is test_row_0/B:col10/1732466445232/Put/seqid=0 2024-11-24T16:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:46,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/2b56a53acfd54d29b33f517b8f37e050 is 50, key is test_row_0/A:col10/1732466445589/Put/seqid=0 2024-11-24T16:40:46,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741894_1070 (size=12949) 2024-11-24T16:40:46,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741895_1071 (size=12949) 2024-11-24T16:40:46,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741896_1072 (size=12301) 2024-11-24T16:40:46,101 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/2b56a53acfd54d29b33f517b8f37e050 2024-11-24T16:40:46,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fd7f30722da848f7a1254c29b74ac63d is 50, key is test_row_0/B:col10/1732466445589/Put/seqid=0 2024-11-24T16:40:46,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741897_1073 (size=12301) 2024-11-24T16:40:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T16:40:46,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466446263, max timestamp=1732466446297Num uniques 859; 2024-11-24T16:40:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:40:46,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466446263, max timestamp=1732466446297Num uniques 859; 2024-11-24T16:40:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466506299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:40:46,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466446263, max timestamp=1732466446297Num uniques 859; 2024-11-24T16:40:46,302 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:40:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:46,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466506377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466506403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:46,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466506483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:46,500 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/eba52c95704f41119c2264e5d4dfb78e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eba52c95704f41119c2264e5d4dfb78e 2024-11-24T16:40:46,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/ec00ce162b7e4cfc8ce81954b97d0de5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ec00ce162b7e4cfc8ce81954b97d0de5 2024-11-24T16:40:46,509 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into eba52c95704f41119c2264e5d4dfb78e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:46,509 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:46,509 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=13, startTime=1732466446057; duration=0sec 2024-11-24T16:40:46,510 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:46,510 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:46,510 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:46,512 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into ec00ce162b7e4cfc8ce81954b97d0de5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:46,512 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:46,512 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:46,512 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=13, startTime=1732466446058; duration=0sec 2024-11-24T16:40:46,512 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:46,513 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:46,513 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:46,513 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:46,513 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1c3be3ab8cd046e59ddbe359c10dfc66, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/2d3bc21f6e774c40b39900047959b267, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3e4e3699ebc64ab68b8c25291c67b87e] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=36.6 K 2024-11-24T16:40:46,513 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3be3ab8cd046e59ddbe359c10dfc66, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732466444001 2024-11-24T16:40:46,514 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d3bc21f6e774c40b39900047959b267, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732466444361 2024-11-24T16:40:46,515 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4e3699ebc64ab68b8c25291c67b87e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466445232 2024-11-24T16:40:46,523 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fd7f30722da848f7a1254c29b74ac63d 2024-11-24T16:40:46,524 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:46,525 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3efb0b5903fc462580705fae608d21bd is 50, key is test_row_0/C:col10/1732466445232/Put/seqid=0 2024-11-24T16:40:46,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d645fd7fadef4715a8e1a50e4fd746ae is 50, key is test_row_0/C:col10/1732466445589/Put/seqid=0 2024-11-24T16:40:46,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741898_1074 (size=12949) 2024-11-24T16:40:46,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741899_1075 (size=12301) 2024-11-24T16:40:46,545 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3efb0b5903fc462580705fae608d21bd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3efb0b5903fc462580705fae608d21bd 2024-11-24T16:40:46,545 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d645fd7fadef4715a8e1a50e4fd746ae 2024-11-24T16:40:46,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/2b56a53acfd54d29b33f517b8f37e050 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/2b56a53acfd54d29b33f517b8f37e050 2024-11-24T16:40:46,554 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into 3efb0b5903fc462580705fae608d21bd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:46,554 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:46,554 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=13, startTime=1732466446058; duration=0sec 2024-11-24T16:40:46,554 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:46,554 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:46,561 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/2b56a53acfd54d29b33f517b8f37e050, entries=150, sequenceid=334, filesize=12.0 K 2024-11-24T16:40:46,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/fd7f30722da848f7a1254c29b74ac63d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd7f30722da848f7a1254c29b74ac63d 2024-11-24T16:40:46,567 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd7f30722da848f7a1254c29b74ac63d, entries=150, sequenceid=334, filesize=12.0 K 2024-11-24T16:40:46,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d645fd7fadef4715a8e1a50e4fd746ae as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d645fd7fadef4715a8e1a50e4fd746ae 2024-11-24T16:40:46,574 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d645fd7fadef4715a8e1a50e4fd746ae, entries=150, sequenceid=334, filesize=12.0 K 2024-11-24T16:40:46,575 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 950b8a656f36ed921dc512c3b64e83ba in 492ms, sequenceid=334, compaction requested=false 2024-11-24T16:40:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:46,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:46,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-24T16:40:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-24T16:40:46,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T16:40:46,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 958 msec 2024-11-24T16:40:46,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 966 msec 2024-11-24T16:40:46,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:46,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-24T16:40:46,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:46,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:46,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4cb55f2532864f12b346ee9e67bc46c4 is 50, key is test_row_0/A:col10/1732466446375/Put/seqid=0 2024-11-24T16:40:46,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741900_1076 (size=14741) 2024-11-24T16:40:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466506652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:46,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466506693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T16:40:46,754 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:46,755 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-24T16:40:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T16:40:46,757 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:46,758 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:46,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:46,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466506764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T16:40:46,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:46,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466506974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:47,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:47,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466507003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:47,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4cb55f2532864f12b346ee9e67bc46c4 2024-11-24T16:40:47,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/95783e5d438342b5892ffc2380bb8a8b is 50, key is test_row_0/B:col10/1732466446375/Put/seqid=0 2024-11-24T16:40:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741901_1077 (size=12301) 2024-11-24T16:40:47,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/95783e5d438342b5892ffc2380bb8a8b 2024-11-24T16:40:47,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/01be69f362fb40978496b4888e565df2 is 50, key is test_row_0/C:col10/1732466446375/Put/seqid=0 2024-11-24T16:40:47,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741902_1078 (size=12301) 2024-11-24T16:40:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T16:40:47,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:47,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:47,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466507284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:47,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T16:40:47,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/01be69f362fb40978496b4888e565df2 2024-11-24T16:40:47,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4cb55f2532864f12b346ee9e67bc46c4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4cb55f2532864f12b346ee9e67bc46c4 2024-11-24T16:40:47,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4cb55f2532864f12b346ee9e67bc46c4, entries=200, sequenceid=373, filesize=14.4 K 2024-11-24T16:40:47,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/95783e5d438342b5892ffc2380bb8a8b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/95783e5d438342b5892ffc2380bb8a8b 2024-11-24T16:40:47,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/95783e5d438342b5892ffc2380bb8a8b, entries=150, sequenceid=373, filesize=12.0 K 2024-11-24T16:40:47,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/01be69f362fb40978496b4888e565df2 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/01be69f362fb40978496b4888e565df2 2024-11-24T16:40:47,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/01be69f362fb40978496b4888e565df2, entries=150, sequenceid=373, filesize=12.0 K 2024-11-24T16:40:47,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 950b8a656f36ed921dc512c3b64e83ba in 886ms, sequenceid=373, compaction requested=true 2024-11-24T16:40:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:47,502 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:47,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:47,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:47,503 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:47,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:47,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:47,504 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:47,504 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:47,504 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,504 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eba52c95704f41119c2264e5d4dfb78e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/2b56a53acfd54d29b33f517b8f37e050, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4cb55f2532864f12b346ee9e67bc46c4] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=39.1 K 2024-11-24T16:40:47,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:47,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:47,504 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,505 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ec00ce162b7e4cfc8ce81954b97d0de5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd7f30722da848f7a1254c29b74ac63d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/95783e5d438342b5892ffc2380bb8a8b] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=36.7 K 2024-11-24T16:40:47,505 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting eba52c95704f41119c2264e5d4dfb78e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466445232 2024-11-24T16:40:47,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ec00ce162b7e4cfc8ce81954b97d0de5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466445232 2024-11-24T16:40:47,505 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b56a53acfd54d29b33f517b8f37e050, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732466445589 2024-11-24T16:40:47,506 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fd7f30722da848f7a1254c29b74ac63d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732466445589 2024-11-24T16:40:47,506 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cb55f2532864f12b346ee9e67bc46c4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732466446356 2024-11-24T16:40:47,506 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 95783e5d438342b5892ffc2380bb8a8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732466446372 2024-11-24T16:40:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:47,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:40:47,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:47,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:47,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:47,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:47,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:47,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,532 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#66 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:47,533 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/3dbea45ffc544433b5344f11a73c8484 is 50, key is test_row_0/A:col10/1732466446375/Put/seqid=0 2024-11-24T16:40:47,536 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:47,537 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/8337b95aa86d46ae8c6bb2870f377ab3 is 50, key is test_row_0/B:col10/1732466446375/Put/seqid=0 2024-11-24T16:40:47,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/302b50ed5ccf46c19afa424afcc902cd is 50, key is test_row_0/A:col10/1732466446650/Put/seqid=0 2024-11-24T16:40:47,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741903_1079 (size=13051) 2024-11-24T16:40:47,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741904_1080 (size=13051) 2024-11-24T16:40:47,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741905_1081 (size=12301) 2024-11-24T16:40:47,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/302b50ed5ccf46c19afa424afcc902cd 2024-11-24T16:40:47,574 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/8337b95aa86d46ae8c6bb2870f377ab3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/8337b95aa86d46ae8c6bb2870f377ab3 2024-11-24T16:40:47,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1907077f78c84a45aed1296a3ec2d4be is 50, key is test_row_0/B:col10/1732466446650/Put/seqid=0 2024-11-24T16:40:47,581 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into 8337b95aa86d46ae8c6bb2870f377ab3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:47,581 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:47,581 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=13, startTime=1732466447502; duration=0sec 2024-11-24T16:40:47,582 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:47,582 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:47,582 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:47,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:47,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:47,583 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,583 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3efb0b5903fc462580705fae608d21bd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d645fd7fadef4715a8e1a50e4fd746ae, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/01be69f362fb40978496b4888e565df2] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=36.7 K 2024-11-24T16:40:47,584 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3efb0b5903fc462580705fae608d21bd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466445232 2024-11-24T16:40:47,584 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d645fd7fadef4715a8e1a50e4fd746ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732466445589 2024-11-24T16:40:47,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741906_1082 (size=12301) 2024-11-24T16:40:47,587 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 01be69f362fb40978496b4888e565df2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732466446372 2024-11-24T16:40:47,601 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:47,602 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/dd8dc12de3904f4a93098d9a824e632f is 50, key is test_row_0/C:col10/1732466446375/Put/seqid=0 2024-11-24T16:40:47,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741907_1083 (size=13051) 2024-11-24T16:40:47,613 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/dd8dc12de3904f4a93098d9a824e632f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/dd8dc12de3904f4a93098d9a824e632f 2024-11-24T16:40:47,621 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into dd8dc12de3904f4a93098d9a824e632f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:47,621 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:47,621 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=13, startTime=1732466447503; duration=0sec 2024-11-24T16:40:47,621 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:47,621 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:47,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466447602, max timestamp=1732466447630Num uniques 859; 2024-11-24T16:40:47,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:40:47,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466447602, max timestamp=1732466447630Num uniques 859; 2024-11-24T16:40:47,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466447602, max timestamp=1732466447630Num uniques 859; 2024-11-24T16:40:47,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:40:47,636 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:40:47,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:47,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:47,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466507711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466507803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466507824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:47,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:47,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T16:40:47,970 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/3dbea45ffc544433b5344f11a73c8484 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/3dbea45ffc544433b5344f11a73c8484 2024-11-24T16:40:47,976 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into 3dbea45ffc544433b5344f11a73c8484(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:47,977 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:47,977 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=13, startTime=1732466447502; duration=0sec 2024-11-24T16:40:47,977 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:47,977 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:47,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1907077f78c84a45aed1296a3ec2d4be 2024-11-24T16:40:47,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:47,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:47,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:47,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:47,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3474908dc8974db491450d3d35bf710e is 50, key is test_row_0/C:col10/1732466446650/Put/seqid=0 2024-11-24T16:40:48,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741908_1084 (size=12301) 2024-11-24T16:40:48,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3474908dc8974db491450d3d35bf710e 2024-11-24T16:40:48,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/302b50ed5ccf46c19afa424afcc902cd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/302b50ed5ccf46c19afa424afcc902cd 2024-11-24T16:40:48,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/302b50ed5ccf46c19afa424afcc902cd, entries=150, sequenceid=383, filesize=12.0 K 2024-11-24T16:40:48,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1907077f78c84a45aed1296a3ec2d4be as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1907077f78c84a45aed1296a3ec2d4be 2024-11-24T16:40:48,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1907077f78c84a45aed1296a3ec2d4be, entries=150, sequenceid=383, filesize=12.0 K 2024-11-24T16:40:48,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3474908dc8974db491450d3d35bf710e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3474908dc8974db491450d3d35bf710e 2024-11-24T16:40:48,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466508033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:48,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3474908dc8974db491450d3d35bf710e, entries=150, sequenceid=383, filesize=12.0 K 2024-11-24T16:40:48,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 950b8a656f36ed921dc512c3b64e83ba in 514ms, sequenceid=383, compaction requested=false 2024-11-24T16:40:48,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:48,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T16:40:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:48,148 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T16:40:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:48,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/54057cef397e4512a62689732efc4eed is 50, key is test_row_0/A:col10/1732466447664/Put/seqid=0 2024-11-24T16:40:48,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741909_1085 (size=12301) 2024-11-24T16:40:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:48,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:48,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466508378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:48,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466508494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:48,561 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/54057cef397e4512a62689732efc4eed 2024-11-24T16:40:48,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/c082a2d885094b83aae6b71eae152c99 is 50, key is test_row_0/B:col10/1732466447664/Put/seqid=0 2024-11-24T16:40:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741910_1086 (size=12301) 2024-11-24T16:40:48,578 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/c082a2d885094b83aae6b71eae152c99 2024-11-24T16:40:48,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8 is 50, key is test_row_0/C:col10/1732466447664/Put/seqid=0 2024-11-24T16:40:48,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741911_1087 (size=12301) 2024-11-24T16:40:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466508704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:48,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:48,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466508814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:48,818 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:48,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T16:40:48,992 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8 2024-11-24T16:40:48,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/54057cef397e4512a62689732efc4eed as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/54057cef397e4512a62689732efc4eed 2024-11-24T16:40:49,004 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/54057cef397e4512a62689732efc4eed, entries=150, sequenceid=421, filesize=12.0 K 2024-11-24T16:40:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/c082a2d885094b83aae6b71eae152c99 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/c082a2d885094b83aae6b71eae152c99 2024-11-24T16:40:49,010 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/c082a2d885094b83aae6b71eae152c99, entries=150, sequenceid=421, filesize=12.0 K 2024-11-24T16:40:49,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8 2024-11-24T16:40:49,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466509014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:49,017 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8, entries=150, sequenceid=421, filesize=12.0 K 2024-11-24T16:40:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:49,018 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 950b8a656f36ed921dc512c3b64e83ba in 870ms, sequenceid=421, compaction requested=true 2024-11-24T16:40:49,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:49,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:49,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-24T16:40:49,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-24T16:40:49,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T16:40:49,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2620 sec 2024-11-24T16:40:49,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2670 sec 2024-11-24T16:40:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:49,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:40:49,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:49,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:49,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:49,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:49,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:49,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:49,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/d8475eb0a8e2477cb94ebe380ed27690 is 50, key is test_row_0/A:col10/1732466448344/Put/seqid=0 2024-11-24T16:40:49,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741912_1088 (size=12301) 2024-11-24T16:40:49,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466449589, max timestamp=1732466449642Num uniques 859; 2024-11-24T16:40:49,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:40:49,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466449589, max timestamp=1732466449642Num uniques 859; 2024-11-24T16:40:49,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:40:49,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466449589, max timestamp=1732466449642Num uniques 859; 2024-11-24T16:40:49,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:40:49,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466509739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:49,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56336 deadline: 1732466509803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:49,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56294 deadline: 1732466509804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:49,811 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:49,812 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:49,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56326 deadline: 1732466509814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:49,819 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:49,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466509853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:49,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/d8475eb0a8e2477cb94ebe380ed27690 2024-11-24T16:40:49,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/774e3e526cfd4937966c1641c3e806ee is 50, key is test_row_0/B:col10/1732466448344/Put/seqid=0 2024-11-24T16:40:49,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741913_1089 (size=12301) 2024-11-24T16:40:50,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466510063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:50,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/774e3e526cfd4937966c1641c3e806ee 2024-11-24T16:40:50,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:50,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466510374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:50,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad is 50, key is test_row_0/C:col10/1732466448344/Put/seqid=0 2024-11-24T16:40:50,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741914_1090 (size=12301) 2024-11-24T16:40:50,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad 2024-11-24T16:40:50,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/d8475eb0a8e2477cb94ebe380ed27690 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d8475eb0a8e2477cb94ebe380ed27690 2024-11-24T16:40:50,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d8475eb0a8e2477cb94ebe380ed27690, entries=150, sequenceid=432, filesize=12.0 K 2024-11-24T16:40:50,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/774e3e526cfd4937966c1641c3e806ee as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/774e3e526cfd4937966c1641c3e806ee 2024-11-24T16:40:50,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/774e3e526cfd4937966c1641c3e806ee, entries=150, sequenceid=432, filesize=12.0 K 2024-11-24T16:40:50,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad 2024-11-24T16:40:50,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad, entries=150, sequenceid=432, filesize=12.0 K 2024-11-24T16:40:50,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 950b8a656f36ed921dc512c3b64e83ba in 946ms, sequenceid=432, compaction requested=true 2024-11-24T16:40:50,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:50,475 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:50,477 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:50,477 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:50,477 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:50,477 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/3dbea45ffc544433b5344f11a73c8484, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/302b50ed5ccf46c19afa424afcc902cd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/54057cef397e4512a62689732efc4eed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d8475eb0a8e2477cb94ebe380ed27690] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=48.8 K 2024-11-24T16:40:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:50,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:50,478 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:50,478 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dbea45ffc544433b5344f11a73c8484, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732466446372 2024-11-24T16:40:50,479 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302b50ed5ccf46c19afa424afcc902cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732466446636 2024-11-24T16:40:50,479 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54057cef397e4512a62689732efc4eed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732466447664 2024-11-24T16:40:50,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:50,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:50,480 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8475eb0a8e2477cb94ebe380ed27690, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466448344 2024-11-24T16:40:50,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:50,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:50,481 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:50,481 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:50,481 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:50,481 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/8337b95aa86d46ae8c6bb2870f377ab3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1907077f78c84a45aed1296a3ec2d4be, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/c082a2d885094b83aae6b71eae152c99, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/774e3e526cfd4937966c1641c3e806ee] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=48.8 K 2024-11-24T16:40:50,482 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 8337b95aa86d46ae8c6bb2870f377ab3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732466446372 2024-11-24T16:40:50,482 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1907077f78c84a45aed1296a3ec2d4be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732466446636 2024-11-24T16:40:50,484 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c082a2d885094b83aae6b71eae152c99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732466447664 2024-11-24T16:40:50,485 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 774e3e526cfd4937966c1641c3e806ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466448344 2024-11-24T16:40:50,500 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:50,501 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/96654314ba3f4412a20e2ad92aba8718 is 50, key is test_row_0/A:col10/1732466448344/Put/seqid=0 2024-11-24T16:40:50,502 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#79 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:50,503 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/3b4b15f9429b4b20b3fbb2a64ffcca15 is 50, key is test_row_0/B:col10/1732466448344/Put/seqid=0 2024-11-24T16:40:50,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741916_1092 (size=13187) 2024-11-24T16:40:50,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741915_1091 (size=13187) 2024-11-24T16:40:50,563 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/96654314ba3f4412a20e2ad92aba8718 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/96654314ba3f4412a20e2ad92aba8718 2024-11-24T16:40:50,574 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into 96654314ba3f4412a20e2ad92aba8718(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:50,575 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:50,575 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=12, startTime=1732466450475; duration=0sec 2024-11-24T16:40:50,575 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:50,575 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:50,576 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:40:50,582 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:40:50,582 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:50,582 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:50,582 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/dd8dc12de3904f4a93098d9a824e632f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3474908dc8974db491450d3d35bf710e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=48.8 K 2024-11-24T16:40:50,583 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8dc12de3904f4a93098d9a824e632f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732466446372 2024-11-24T16:40:50,585 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3474908dc8974db491450d3d35bf710e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732466446636 2024-11-24T16:40:50,585 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7feb33b1e7d4b0fb8a85bd3eeeddbc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732466447664 2024-11-24T16:40:50,586 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f60f4fad65f41bfb7a5ac5aa2d2d0ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466448344 2024-11-24T16:40:50,603 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:50,603 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/09eeda0c2a58466f875c74ed077a7f67 is 50, key is test_row_0/C:col10/1732466448344/Put/seqid=0 2024-11-24T16:40:50,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741917_1093 (size=13187) 2024-11-24T16:40:50,631 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/09eeda0c2a58466f875c74ed077a7f67 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/09eeda0c2a58466f875c74ed077a7f67 2024-11-24T16:40:50,660 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into 09eeda0c2a58466f875c74ed077a7f67(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:50,660 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:50,660 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=12, startTime=1732466450480; duration=0sec 2024-11-24T16:40:50,661 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:50,661 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:50,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T16:40:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:50,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:50,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:50,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:50,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:50,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/64c62d98ff154b34bf27965c6b5c55fe is 50, key is test_row_0/A:col10/1732466449736/Put/seqid=0 2024-11-24T16:40:50,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741918_1094 (size=14741) 2024-11-24T16:40:50,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/64c62d98ff154b34bf27965c6b5c55fe 2024-11-24T16:40:50,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466510885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:50,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466510894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T16:40:50,915 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:50,915 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-24T16:40:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T16:40:50,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:50,921 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:50,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:50,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b83a953a8ede498db62022513193be0b is 50, key is test_row_0/B:col10/1732466449736/Put/seqid=0 2024-11-24T16:40:50,927 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/3b4b15f9429b4b20b3fbb2a64ffcca15 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/3b4b15f9429b4b20b3fbb2a64ffcca15 2024-11-24T16:40:50,936 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into 3b4b15f9429b4b20b3fbb2a64ffcca15(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:50,936 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:50,936 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=12, startTime=1732466450478; duration=0sec 2024-11-24T16:40:50,939 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:50,939 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:50,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741919_1095 (size=12301) 2024-11-24T16:40:50,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b83a953a8ede498db62022513193be0b 2024-11-24T16:40:50,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3ceaa7b3969b4bf69bac110b014f6dd0 is 50, key is test_row_0/C:col10/1732466449736/Put/seqid=0 2024-11-24T16:40:51,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466511004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:51,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T16:40:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741920_1096 (size=12301) 2024-11-24T16:40:51,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3ceaa7b3969b4bf69bac110b014f6dd0 2024-11-24T16:40:51,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/64c62d98ff154b34bf27965c6b5c55fe as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/64c62d98ff154b34bf27965c6b5c55fe 2024-11-24T16:40:51,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/64c62d98ff154b34bf27965c6b5c55fe, entries=200, sequenceid=470, filesize=14.4 K 2024-11-24T16:40:51,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b83a953a8ede498db62022513193be0b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b83a953a8ede498db62022513193be0b 2024-11-24T16:40:51,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b83a953a8ede498db62022513193be0b, entries=150, sequenceid=470, filesize=12.0 K 2024-11-24T16:40:51,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/3ceaa7b3969b4bf69bac110b014f6dd0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3ceaa7b3969b4bf69bac110b014f6dd0 2024-11-24T16:40:51,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3ceaa7b3969b4bf69bac110b014f6dd0, entries=150, sequenceid=470, filesize=12.0 K 2024-11-24T16:40:51,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 950b8a656f36ed921dc512c3b64e83ba in 221ms, sequenceid=470, compaction requested=false 2024-11-24T16:40:51,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:51,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T16:40:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:51,075 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:40:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:51,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:51,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/925ddd4ce381414baec9df663e9b252b is 50, key is test_row_0/A:col10/1732466450888/Put/seqid=0 2024-11-24T16:40:51,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741921_1097 (size=12301) 2024-11-24T16:40:51,116 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/925ddd4ce381414baec9df663e9b252b 2024-11-24T16:40:51,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1906e94f72a344aba43013f0f47c0879 is 50, key is test_row_0/B:col10/1732466450888/Put/seqid=0 2024-11-24T16:40:51,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741922_1098 (size=12301) 2024-11-24T16:40:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:51,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T16:40:51,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466451331, max timestamp=1732466451403Num uniques 859; 2024-11-24T16:40:51,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466451331, max timestamp=1732466451403Num uniques 859; 2024-11-24T16:40:51,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:51,407 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:51,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466451331, max timestamp=1732466451403Num uniques 859; 2024-11-24T16:40:51,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:51,542 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1906e94f72a344aba43013f0f47c0879 2024-11-24T16:40:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T16:40:51,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/f9584e51b7a64da99551189fd6a688b1 is 50, key is test_row_0/C:col10/1732466450888/Put/seqid=0 2024-11-24T16:40:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466511569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:51,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741923_1099 (size=12301) 2024-11-24T16:40:51,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466511684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:51,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466511893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:51,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:51,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466511905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:51,916 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:51,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:51,982 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/f9584e51b7a64da99551189fd6a688b1 2024-11-24T16:40:51,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/925ddd4ce381414baec9df663e9b252b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/925ddd4ce381414baec9df663e9b252b 2024-11-24T16:40:51,995 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/925ddd4ce381414baec9df663e9b252b, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T16:40:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/1906e94f72a344aba43013f0f47c0879 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1906e94f72a344aba43013f0f47c0879 2024-11-24T16:40:52,004 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1906e94f72a344aba43013f0f47c0879, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T16:40:52,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/f9584e51b7a64da99551189fd6a688b1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f9584e51b7a64da99551189fd6a688b1 2024-11-24T16:40:52,012 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f9584e51b7a64da99551189fd6a688b1, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T16:40:52,014 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 950b8a656f36ed921dc512c3b64e83ba in 939ms, sequenceid=481, compaction requested=true 2024-11-24T16:40:52,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:52,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:52,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-24T16:40:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-24T16:40:52,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T16:40:52,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0950 sec 2024-11-24T16:40:52,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.1040 sec 2024-11-24T16:40:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T16:40:52,054 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:52,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-24T16:40:52,058 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:52,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T16:40:52,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:52,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T16:40:52,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466451543, max timestamp=1732466452208Num uniques 1018; 2024-11-24T16:40:52,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466451543, max timestamp=1732466452208Num uniques 1018; 2024-11-24T16:40:52,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:40:52,214 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:40:52,217 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466451543, max timestamp=1732466452208Num uniques 1018; 2024-11-24T16:40:52,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:40:52,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T16:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:52,219 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-24T16:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:52,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/22e0c9598d524afc843d6a9caa96c040 is 50, key is test_row_0/A:col10/1732466452208/Put/seqid=0 2024-11-24T16:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:52,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:52,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741924_1100 (size=14741) 2024-11-24T16:40:52,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466452344, max timestamp=1732466452370Num uniques 859; 2024-11-24T16:40:52,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466452344, max timestamp=1732466452370Num uniques 859; 2024-11-24T16:40:52,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:52,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:52,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466452344, max timestamp=1732466452370Num uniques 859; 2024-11-24T16:40:52,380 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:40:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T16:40:52,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466452508, max timestamp=1732466452538Num uniques 1018; 2024-11-24T16:40:52,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:40:52,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466452508, max timestamp=1732466452538Num uniques 1018; 2024-11-24T16:40:52,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466452508, max timestamp=1732466452538Num uniques 1018; 2024-11-24T16:40:52,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:40:52,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:40:52,679 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/22e0c9598d524afc843d6a9caa96c040 2024-11-24T16:40:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T16:40:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b1a9276e51bf4f0599b531aae230de45 is 50, key is test_row_0/B:col10/1732466452208/Put/seqid=0 2024-11-24T16:40:52,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732466452594, max timestamp=1732466452698Num uniques 1027; 2024-11-24T16:40:52,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T16:40:52,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732466452594, max timestamp=1732466452698Num uniques 1027; 2024-11-24T16:40:52,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T16:40:52,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732466452594, max timestamp=1732466452698Num uniques 1027; 2024-11-24T16:40:52,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T16:40:52,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741925_1101 (size=12301) 2024-11-24T16:40:52,720 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b1a9276e51bf4f0599b531aae230de45 2024-11-24T16:40:52,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/00afe6acf3294801a4bffa855bfd1df9 is 50, key is test_row_0/C:col10/1732466452208/Put/seqid=0 2024-11-24T16:40:52,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741926_1102 (size=12301) 2024-11-24T16:40:52,747 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/00afe6acf3294801a4bffa855bfd1df9 2024-11-24T16:40:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/22e0c9598d524afc843d6a9caa96c040 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/22e0c9598d524afc843d6a9caa96c040 2024-11-24T16:40:52,766 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/22e0c9598d524afc843d6a9caa96c040, entries=200, sequenceid=519, filesize=14.4 K 2024-11-24T16:40:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/b1a9276e51bf4f0599b531aae230de45 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b1a9276e51bf4f0599b531aae230de45 2024-11-24T16:40:52,777 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b1a9276e51bf4f0599b531aae230de45, entries=150, sequenceid=519, filesize=12.0 K 2024-11-24T16:40:52,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/00afe6acf3294801a4bffa855bfd1df9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/00afe6acf3294801a4bffa855bfd1df9 2024-11-24T16:40:52,785 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/00afe6acf3294801a4bffa855bfd1df9, entries=150, sequenceid=519, filesize=12.0 K 2024-11-24T16:40:52,787 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=100.63 KB/103050 for 950b8a656f36ed921dc512c3b64e83ba in 567ms, sequenceid=519, compaction requested=true 2024-11-24T16:40:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-24T16:40:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-24T16:40:52,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=114.05 KB heapSize=318.84 KB 2024-11-24T16:40:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:40:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:52,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T16:40:52,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 724 msec 2024-11-24T16:40:52,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 738 msec 2024-11-24T16:40:52,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/b19caab6ecbd47a9b3d0a0b3553774d1 is 50, key is test_row_0/A:col10/1732466452759/Put/seqid=0 2024-11-24T16:40:52,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741927_1103 (size=14741) 2024-11-24T16:40:52,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/b19caab6ecbd47a9b3d0a0b3553774d1 2024-11-24T16:40:52,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/16c57baefbcd4f8b9d41a6c35bfd27df is 50, key is test_row_0/B:col10/1732466452759/Put/seqid=0 2024-11-24T16:40:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466512898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:52,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741928_1104 (size=12301) 2024-11-24T16:40:52,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/16c57baefbcd4f8b9d41a6c35bfd27df 2024-11-24T16:40:52,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9 is 50, key is test_row_0/C:col10/1732466452759/Put/seqid=0 2024-11-24T16:40:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741929_1105 (size=12301) 2024-11-24T16:40:52,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9 2024-11-24T16:40:52,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/b19caab6ecbd47a9b3d0a0b3553774d1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/b19caab6ecbd47a9b3d0a0b3553774d1 2024-11-24T16:40:52,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/b19caab6ecbd47a9b3d0a0b3553774d1, entries=200, sequenceid=586, filesize=14.4 K 2024-11-24T16:40:52,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/16c57baefbcd4f8b9d41a6c35bfd27df as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/16c57baefbcd4f8b9d41a6c35bfd27df 2024-11-24T16:40:52,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/16c57baefbcd4f8b9d41a6c35bfd27df, entries=150, sequenceid=586, filesize=12.0 K 2024-11-24T16:40:52,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9 2024-11-24T16:40:52,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9, entries=150, sequenceid=586, filesize=12.0 K 2024-11-24T16:40:52,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~318.80 KB/326448, currentSize=67.09 KB/68700 for 950b8a656f36ed921dc512c3b64e83ba in 185ms, sequenceid=586, compaction requested=true 2024-11-24T16:40:52,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:52,976 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T16:40:52,976 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T16:40:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:52,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:52,978 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 69711 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T16:40:52,978 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:52,978 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:52,978 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/96654314ba3f4412a20e2ad92aba8718, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/64c62d98ff154b34bf27965c6b5c55fe, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/925ddd4ce381414baec9df663e9b252b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/22e0c9598d524afc843d6a9caa96c040, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/b19caab6ecbd47a9b3d0a0b3553774d1] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=68.1 K 2024-11-24T16:40:52,979 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62391 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T16:40:52,979 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:52,979 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:52,979 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/3b4b15f9429b4b20b3fbb2a64ffcca15, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b83a953a8ede498db62022513193be0b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1906e94f72a344aba43013f0f47c0879, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b1a9276e51bf4f0599b531aae230de45, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/16c57baefbcd4f8b9d41a6c35bfd27df] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=60.9 K 2024-11-24T16:40:52,979 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96654314ba3f4412a20e2ad92aba8718, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466448344 2024-11-24T16:40:52,979 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b4b15f9429b4b20b3fbb2a64ffcca15, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466448344 2024-11-24T16:40:52,980 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64c62d98ff154b34bf27965c6b5c55fe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732466449706 2024-11-24T16:40:52,980 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting b83a953a8ede498db62022513193be0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732466449706 2024-11-24T16:40:52,980 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 925ddd4ce381414baec9df663e9b252b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466450849 2024-11-24T16:40:52,981 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1906e94f72a344aba43013f0f47c0879, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466450849 2024-11-24T16:40:52,981 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22e0c9598d524afc843d6a9caa96c040, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732466451543 2024-11-24T16:40:52,984 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b19caab6ecbd47a9b3d0a0b3553774d1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1732466452736 2024-11-24T16:40:52,988 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting b1a9276e51bf4f0599b531aae230de45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732466451543 2024-11-24T16:40:52,990 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c57baefbcd4f8b9d41a6c35bfd27df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1732466452736 2024-11-24T16:40:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:53,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T16:40:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:53,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:53,010 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#93 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:53,011 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/d5d6329e12c44fc9a2d1398464892b9a is 50, key is test_row_0/A:col10/1732466452759/Put/seqid=0 2024-11-24T16:40:53,024 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#94 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:53,026 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/cc914c1aaf0e45fe811d9612b1969ebf is 50, key is test_row_0/B:col10/1732466452759/Put/seqid=0 2024-11-24T16:40:53,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4a4aeda7dcce4623aa3910615b98e9a2 is 50, key is test_row_0/A:col10/1732466453004/Put/seqid=0 2024-11-24T16:40:53,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741930_1106 (size=13357) 2024-11-24T16:40:53,073 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/d5d6329e12c44fc9a2d1398464892b9a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d5d6329e12c44fc9a2d1398464892b9a 2024-11-24T16:40:53,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741932_1108 (size=14741) 2024-11-24T16:40:53,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4a4aeda7dcce4623aa3910615b98e9a2 2024-11-24T16:40:53,084 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into d5d6329e12c44fc9a2d1398464892b9a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:53,085 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:53,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741931_1107 (size=13357) 2024-11-24T16:40:53,085 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=11, startTime=1732466452976; duration=0sec 2024-11-24T16:40:53,085 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:53,085 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:53,085 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T16:40:53,107 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62391 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T16:40:53,108 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:53,108 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,108 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/09eeda0c2a58466f875c74ed077a7f67, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3ceaa7b3969b4bf69bac110b014f6dd0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f9584e51b7a64da99551189fd6a688b1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/00afe6acf3294801a4bffa855bfd1df9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=60.9 K 2024-11-24T16:40:53,109 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09eeda0c2a58466f875c74ed077a7f67, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466448344 2024-11-24T16:40:53,111 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ceaa7b3969b4bf69bac110b014f6dd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732466449706 2024-11-24T16:40:53,116 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9584e51b7a64da99551189fd6a688b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466450849 2024-11-24T16:40:53,117 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00afe6acf3294801a4bffa855bfd1df9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732466451543 2024-11-24T16:40:53,118 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95f5c95f2fbe4a50a3e7f4aa770b6ef9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1732466452736 2024-11-24T16:40:53,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/adf4a81d5a504675a5fd511cf8ec1b7c is 50, key is test_row_0/B:col10/1732466453004/Put/seqid=0 2024-11-24T16:40:53,140 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#97 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:53,141 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/9aa0f1bedfd449d29adcd2632a203aae is 50, key is test_row_0/C:col10/1732466452759/Put/seqid=0 2024-11-24T16:40:53,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741933_1109 (size=12301) 2024-11-24T16:40:53,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/adf4a81d5a504675a5fd511cf8ec1b7c 2024-11-24T16:40:53,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741934_1110 (size=13357) 2024-11-24T16:40:53,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466513152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:53,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/bd4193b3713e4bc5a9c3eb0b1f369dd5 is 50, key is test_row_0/C:col10/1732466453004/Put/seqid=0 2024-11-24T16:40:53,176 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/9aa0f1bedfd449d29adcd2632a203aae as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9aa0f1bedfd449d29adcd2632a203aae 2024-11-24T16:40:53,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741935_1111 (size=12301) 2024-11-24T16:40:53,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=600 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/bd4193b3713e4bc5a9c3eb0b1f369dd5 2024-11-24T16:40:53,185 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into 9aa0f1bedfd449d29adcd2632a203aae(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:53,185 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:53,185 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=11, startTime=1732466452976; duration=0sec 2024-11-24T16:40:53,185 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:53,185 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:53,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/4a4aeda7dcce4623aa3910615b98e9a2 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4a4aeda7dcce4623aa3910615b98e9a2 2024-11-24T16:40:53,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4a4aeda7dcce4623aa3910615b98e9a2, entries=200, sequenceid=600, filesize=14.4 K 2024-11-24T16:40:53,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/adf4a81d5a504675a5fd511cf8ec1b7c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/adf4a81d5a504675a5fd511cf8ec1b7c 2024-11-24T16:40:53,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/adf4a81d5a504675a5fd511cf8ec1b7c, entries=150, sequenceid=600, filesize=12.0 K 2024-11-24T16:40:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T16:40:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/bd4193b3713e4bc5a9c3eb0b1f369dd5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/bd4193b3713e4bc5a9c3eb0b1f369dd5 2024-11-24T16:40:53,205 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:53,206 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-24T16:40:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T16:40:53,213 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:53,214 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:53,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:53,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/bd4193b3713e4bc5a9c3eb0b1f369dd5, entries=150, sequenceid=600, filesize=12.0 K 2024-11-24T16:40:53,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 950b8a656f36ed921dc512c3b64e83ba in 221ms, sequenceid=600, compaction requested=false 2024-11-24T16:40:53,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:53,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T16:40:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:53,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:53,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:53,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:53,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:53,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/6c804c7b7afc451e8e824294004d6f35 is 50, key is test_row_0/A:col10/1732466453146/Put/seqid=0 2024-11-24T16:40:53,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T16:40:53,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741936_1112 (size=14741) 2024-11-24T16:40:53,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T16:40:53,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:53,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:53,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466513362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:53,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466513474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:53,497 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/cc914c1aaf0e45fe811d9612b1969ebf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc914c1aaf0e45fe811d9612b1969ebf 2024-11-24T16:40:53,506 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into cc914c1aaf0e45fe811d9612b1969ebf(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:53,507 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:53,507 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=11, startTime=1732466452976; duration=0sec 2024-11-24T16:40:53,507 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:53,507 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:53,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T16:40:53,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:53,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T16:40:53,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T16:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:53,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466513694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:53,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/6c804c7b7afc451e8e824294004d6f35 2024-11-24T16:40:53,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/acca177596524c71bd47744e0eaf7786 is 50, key is test_row_0/B:col10/1732466453146/Put/seqid=0 2024-11-24T16:40:53,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741937_1113 (size=12301) 2024-11-24T16:40:53,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/acca177596524c71bd47744e0eaf7786 2024-11-24T16:40:53,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/904a961326c24f3b82002053c1a19937 is 50, key is test_row_0/C:col10/1732466453146/Put/seqid=0 2024-11-24T16:40:53,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741938_1114 (size=12301) 2024-11-24T16:40:53,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T16:40:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T16:40:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56308 deadline: 1732466513944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:53,952 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 950b8a656f36ed921dc512c3b64e83ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:40:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:53,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T16:40:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:53,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:53,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:54,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466514004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:54,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T16:40:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:54,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:54,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/904a961326c24f3b82002053c1a19937 2024-11-24T16:40:54,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/6c804c7b7afc451e8e824294004d6f35 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6c804c7b7afc451e8e824294004d6f35 2024-11-24T16:40:54,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6c804c7b7afc451e8e824294004d6f35, entries=200, sequenceid=621, filesize=14.4 K 2024-11-24T16:40:54,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/acca177596524c71bd47744e0eaf7786 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/acca177596524c71bd47744e0eaf7786 2024-11-24T16:40:54,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/acca177596524c71bd47744e0eaf7786, entries=150, sequenceid=621, filesize=12.0 K 2024-11-24T16:40:54,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/904a961326c24f3b82002053c1a19937 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/904a961326c24f3b82002053c1a19937 2024-11-24T16:40:54,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/904a961326c24f3b82002053c1a19937, entries=150, sequenceid=621, filesize=12.0 K 2024-11-24T16:40:54,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 950b8a656f36ed921dc512c3b64e83ba in 979ms, sequenceid=621, compaction requested=true 2024-11-24T16:40:54,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:54,264 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:54,265 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:54,265 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:54,266 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:54,266 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d5d6329e12c44fc9a2d1398464892b9a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4a4aeda7dcce4623aa3910615b98e9a2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6c804c7b7afc451e8e824294004d6f35] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=41.8 K 2024-11-24T16:40:54,266 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5d6329e12c44fc9a2d1398464892b9a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1732466452736 2024-11-24T16:40:54,267 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4aeda7dcce4623aa3910615b98e9a2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732466452846 2024-11-24T16:40:54,269 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c804c7b7afc451e8e824294004d6f35, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1732466453120 2024-11-24T16:40:54,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:54,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:54,282 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:54,283 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:54,283 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:54,283 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:54,283 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc914c1aaf0e45fe811d9612b1969ebf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/adf4a81d5a504675a5fd511cf8ec1b7c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/acca177596524c71bd47744e0eaf7786] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=37.1 K 2024-11-24T16:40:54,284 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting cc914c1aaf0e45fe811d9612b1969ebf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1732466452736 2024-11-24T16:40:54,288 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting adf4a81d5a504675a5fd511cf8ec1b7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732466452846 2024-11-24T16:40:54,290 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting acca177596524c71bd47744e0eaf7786, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1732466453139 2024-11-24T16:40:54,291 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:54,291 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/eeea36a647cb4c9e8f5db0385be5bf23 is 50, key is test_row_0/A:col10/1732466453146/Put/seqid=0 2024-11-24T16:40:54,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T16:40:54,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:54,301 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:40:54,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:54,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:54,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:54,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:54,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:54,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:54,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:54,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:54,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:54,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:54,320 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/a9e3b1ead2504c4c89fee71f5489e7a2 is 50, key is test_row_0/A:col10/1732466453354/Put/seqid=0 2024-11-24T16:40:54,322 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/683a55884d12444cae26bbc9cc79701d is 50, key is test_row_0/B:col10/1732466453146/Put/seqid=0 2024-11-24T16:40:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T16:40:54,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741939_1115 (size=13459) 2024-11-24T16:40:54,361 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/eeea36a647cb4c9e8f5db0385be5bf23 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eeea36a647cb4c9e8f5db0385be5bf23 2024-11-24T16:40:54,371 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into eeea36a647cb4c9e8f5db0385be5bf23(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:54,371 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:54,371 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=13, startTime=1732466454264; duration=0sec 2024-11-24T16:40:54,372 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:54,372 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:54,372 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:54,373 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:54,373 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:54,373 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:54,373 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9aa0f1bedfd449d29adcd2632a203aae, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/bd4193b3713e4bc5a9c3eb0b1f369dd5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/904a961326c24f3b82002053c1a19937] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=37.1 K 2024-11-24T16:40:54,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741940_1116 (size=12301) 2024-11-24T16:40:54,374 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aa0f1bedfd449d29adcd2632a203aae, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1732466452736 2024-11-24T16:40:54,375 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4193b3713e4bc5a9c3eb0b1f369dd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=600, earliestPutTs=1732466452846 2024-11-24T16:40:54,375 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/a9e3b1ead2504c4c89fee71f5489e7a2 2024-11-24T16:40:54,376 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 904a961326c24f3b82002053c1a19937, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1732466453139 2024-11-24T16:40:54,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741941_1117 (size=13459) 2024-11-24T16:40:54,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/01c3b220df654561b065d285fc7a46ef is 50, key is test_row_0/B:col10/1732466453354/Put/seqid=0 2024-11-24T16:40:54,397 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:54,400 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/6470e1ea2707422a8da490bb147bc8df is 50, key is test_row_0/C:col10/1732466453146/Put/seqid=0 2024-11-24T16:40:54,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741942_1118 (size=12301) 2024-11-24T16:40:54,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741943_1119 (size=13459) 2024-11-24T16:40:54,483 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/6470e1ea2707422a8da490bb147bc8df as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6470e1ea2707422a8da490bb147bc8df 2024-11-24T16:40:54,502 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into 6470e1ea2707422a8da490bb147bc8df(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:54,502 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:54,502 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=13, startTime=1732466454302; duration=0sec 2024-11-24T16:40:54,502 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:54,502 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:54,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:54,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466514654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:54,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:54,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466514774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:54,804 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/683a55884d12444cae26bbc9cc79701d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/683a55884d12444cae26bbc9cc79701d 2024-11-24T16:40:54,811 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into 683a55884d12444cae26bbc9cc79701d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:54,811 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:54,811 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=13, startTime=1732466454282; duration=0sec 2024-11-24T16:40:54,812 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:54,812 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:54,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/01c3b220df654561b065d285fc7a46ef 2024-11-24T16:40:54,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/5fc3521944044c16912fc3a1a8b72a31 is 50, key is test_row_0/C:col10/1732466453354/Put/seqid=0 2024-11-24T16:40:54,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741944_1120 (size=12301) 2024-11-24T16:40:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/5fc3521944044c16912fc3a1a8b72a31 2024-11-24T16:40:54,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/a9e3b1ead2504c4c89fee71f5489e7a2 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a9e3b1ead2504c4c89fee71f5489e7a2 2024-11-24T16:40:54,890 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a9e3b1ead2504c4c89fee71f5489e7a2, entries=150, sequenceid=635, filesize=12.0 K 2024-11-24T16:40:54,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/01c3b220df654561b065d285fc7a46ef as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/01c3b220df654561b065d285fc7a46ef 2024-11-24T16:40:54,928 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/01c3b220df654561b065d285fc7a46ef, entries=150, sequenceid=635, filesize=12.0 K 2024-11-24T16:40:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/5fc3521944044c16912fc3a1a8b72a31 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/5fc3521944044c16912fc3a1a8b72a31 2024-11-24T16:40:54,937 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/5fc3521944044c16912fc3a1a8b72a31, entries=150, sequenceid=635, filesize=12.0 K 2024-11-24T16:40:54,939 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 950b8a656f36ed921dc512c3b64e83ba in 637ms, sequenceid=635, compaction requested=false 2024-11-24T16:40:54,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:54,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:54,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-24T16:40:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-24T16:40:54,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T16:40:54,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7270 sec 2024-11-24T16:40:54,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.7380 sec 2024-11-24T16:40:54,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:40:54,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:54,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:54,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:54,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:54,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:54,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:55,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/85a8da445bfd4e47b80bb4c917e70824 is 50, key is test_row_0/A:col10/1732466454988/Put/seqid=0 2024-11-24T16:40:55,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741945_1121 (size=14741) 2024-11-24T16:40:55,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466515041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:55,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466515154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T16:40:55,366 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:55,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:40:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-24T16:40:55,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466515364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T16:40:55,371 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:40:55,372 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:40:55,372 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:40:55,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/85a8da445bfd4e47b80bb4c917e70824 2024-11-24T16:40:55,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/62083fc2646d4b4bbaa3a2221bae85c8 is 50, key is test_row_0/B:col10/1732466454988/Put/seqid=0 2024-11-24T16:40:55,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741946_1122 (size=12301) 2024-11-24T16:40:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T16:40:55,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T16:40:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:55,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466515674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T16:40:55,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T16:40:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:55,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/62083fc2646d4b4bbaa3a2221bae85c8 2024-11-24T16:40:55,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T16:40:55,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:55,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:55,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:55,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:55,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/80202f14cacd44f39b57ce9ba66a1da2 is 50, key is test_row_0/C:col10/1732466454988/Put/seqid=0 2024-11-24T16:40:55,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741947_1123 (size=12301) 2024-11-24T16:40:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T16:40:56,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T16:40:56,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T16:40:56,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:56,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:40:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732466516184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 2024-11-24T16:40:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:40:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:40:56,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/80202f14cacd44f39b57ce9ba66a1da2 2024-11-24T16:40:56,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/85a8da445bfd4e47b80bb4c917e70824 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/85a8da445bfd4e47b80bb4c917e70824 2024-11-24T16:40:56,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/85a8da445bfd4e47b80bb4c917e70824, entries=200, sequenceid=658, filesize=14.4 K 2024-11-24T16:40:56,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/62083fc2646d4b4bbaa3a2221bae85c8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/62083fc2646d4b4bbaa3a2221bae85c8 2024-11-24T16:40:56,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T16:40:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:40:56,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/62083fc2646d4b4bbaa3a2221bae85c8, entries=150, sequenceid=658, filesize=12.0 K 2024-11-24T16:40:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/80202f14cacd44f39b57ce9ba66a1da2 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/80202f14cacd44f39b57ce9ba66a1da2 2024-11-24T16:40:56,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/80202f14cacd44f39b57ce9ba66a1da2, entries=150, sequenceid=658, filesize=12.0 K 2024-11-24T16:40:56,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 950b8a656f36ed921dc512c3b64e83ba in 1336ms, sequenceid=658, compaction requested=true 2024-11-24T16:40:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:56,326 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:40:56,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:56,327 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:40:56,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:56,328 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 950b8a656f36ed921dc512c3b64e83ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:40:56,328 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/A is initiating minor compaction (all files) 2024-11-24T16:40:56,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:56,328 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/A in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,328 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eeea36a647cb4c9e8f5db0385be5bf23, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a9e3b1ead2504c4c89fee71f5489e7a2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/85a8da445bfd4e47b80bb4c917e70824] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=39.6 K 2024-11-24T16:40:56,328 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:56,328 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/B is initiating minor compaction (all files) 2024-11-24T16:40:56,328 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/B in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,329 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/683a55884d12444cae26bbc9cc79701d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/01c3b220df654561b065d285fc7a46ef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/62083fc2646d4b4bbaa3a2221bae85c8] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=37.2 K 2024-11-24T16:40:56,329 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 683a55884d12444cae26bbc9cc79701d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1732466453139 2024-11-24T16:40:56,329 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeea36a647cb4c9e8f5db0385be5bf23, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1732466453139 2024-11-24T16:40:56,330 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 01c3b220df654561b065d285fc7a46ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1732466453295 2024-11-24T16:40:56,330 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e3b1ead2504c4c89fee71f5489e7a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1732466453295 2024-11-24T16:40:56,330 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 62083fc2646d4b4bbaa3a2221bae85c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732466454608 2024-11-24T16:40:56,331 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a8da445bfd4e47b80bb4c917e70824, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732466454608 2024-11-24T16:40:56,344 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:56,344 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/854915a04b5b4d509150dc1391b529bf is 50, key is test_row_0/B:col10/1732466454988/Put/seqid=0 2024-11-24T16:40:56,347 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:56,348 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/de4dcb68a35d480a867b8956459a3955 is 50, key is test_row_0/A:col10/1732466454988/Put/seqid=0 2024-11-24T16:40:56,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741949_1125 (size=13561) 2024-11-24T16:40:56,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741948_1124 (size=13561) 2024-11-24T16:40:56,404 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/854915a04b5b4d509150dc1391b529bf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/854915a04b5b4d509150dc1391b529bf 2024-11-24T16:40:56,404 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/de4dcb68a35d480a867b8956459a3955 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/de4dcb68a35d480a867b8956459a3955 2024-11-24T16:40:56,412 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/A of 950b8a656f36ed921dc512c3b64e83ba into de4dcb68a35d480a867b8956459a3955(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:56,412 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:56,412 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/A, priority=13, startTime=1732466456326; duration=0sec 2024-11-24T16:40:56,413 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:40:56,413 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:A 2024-11-24T16:40:56,413 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:40:56,413 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/B of 950b8a656f36ed921dc512c3b64e83ba into 854915a04b5b4d509150dc1391b529bf(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:56,414 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:56,414 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/B, priority=13, startTime=1732466456327; duration=0sec 2024-11-24T16:40:56,414 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:56,414 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:B 2024-11-24T16:40:56,414 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:40:56,414 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 950b8a656f36ed921dc512c3b64e83ba/C is initiating minor compaction (all files) 2024-11-24T16:40:56,414 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 950b8a656f36ed921dc512c3b64e83ba/C in TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,414 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6470e1ea2707422a8da490bb147bc8df, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/5fc3521944044c16912fc3a1a8b72a31, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/80202f14cacd44f39b57ce9ba66a1da2] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp, totalSize=37.2 K 2024-11-24T16:40:56,415 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6470e1ea2707422a8da490bb147bc8df, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=621, earliestPutTs=1732466453139 2024-11-24T16:40:56,415 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fc3521944044c16912fc3a1a8b72a31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1732466453295 2024-11-24T16:40:56,416 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80202f14cacd44f39b57ce9ba66a1da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1732466454608 2024-11-24T16:40:56,429 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 950b8a656f36ed921dc512c3b64e83ba#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:40:56,430 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/0943fc5e09a94226af7b17aaa86e2f7f is 50, key is test_row_0/C:col10/1732466454988/Put/seqid=0 2024-11-24T16:40:56,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741950_1126 (size=13561) 2024-11-24T16:40:56,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T16:40:56,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:56,469 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T16:40:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:40:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:40:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:40:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:40:56,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/efd8f2cbde1d471a8ff86cec986873ce is 50, key is test_row_0/A:col10/1732466455040/Put/seqid=0 2024-11-24T16:40:56,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741951_1127 (size=12301) 2024-11-24T16:40:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T16:40:56,869 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/0943fc5e09a94226af7b17aaa86e2f7f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/0943fc5e09a94226af7b17aaa86e2f7f 2024-11-24T16:40:56,878 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 950b8a656f36ed921dc512c3b64e83ba/C of 950b8a656f36ed921dc512c3b64e83ba into 0943fc5e09a94226af7b17aaa86e2f7f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:40:56,878 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:56,878 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., storeName=950b8a656f36ed921dc512c3b64e83ba/C, priority=13, startTime=1732466456327; duration=0sec 2024-11-24T16:40:56,879 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:40:56,879 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 950b8a656f36ed921dc512c3b64e83ba:C 2024-11-24T16:40:56,885 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-24T16:40:56,885 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-24T16:40:56,885 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:56,885 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:56,886 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-24T16:40:56,886 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:56,888 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-24T16:40:56,888 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:56,890 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,890 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,890 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,890 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,891 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,891 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:56,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:56,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:56,891 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,892 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:56,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:56,902 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/efd8f2cbde1d471a8ff86cec986873ce 2024-11-24T16:40:56,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/e8ed64e767a14adaad485afabe9eb845 is 50, key is test_row_0/B:col10/1732466455040/Put/seqid=0 2024-11-24T16:40:56,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741952_1128 (size=12301) 2024-11-24T16:40:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:40:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. as already flushing 2024-11-24T16:40:57,206 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-24T16:40:57,206 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:57,206 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:57,206 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:57,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:57,315 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/e8ed64e767a14adaad485afabe9eb845 2024-11-24T16:40:57,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/1217d6c3992d469ba62719d334db9209 is 50, key is test_row_0/C:col10/1732466455040/Put/seqid=0 2024-11-24T16:40:57,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741953_1129 (size=12301) 2024-11-24T16:40:57,331 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/1217d6c3992d469ba62719d334db9209 2024-11-24T16:40:57,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/efd8f2cbde1d471a8ff86cec986873ce as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/efd8f2cbde1d471a8ff86cec986873ce 2024-11-24T16:40:57,343 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/efd8f2cbde1d471a8ff86cec986873ce, entries=150, sequenceid=672, filesize=12.0 K 2024-11-24T16:40:57,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/e8ed64e767a14adaad485afabe9eb845 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/e8ed64e767a14adaad485afabe9eb845 2024-11-24T16:40:57,350 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/e8ed64e767a14adaad485afabe9eb845, entries=150, sequenceid=672, filesize=12.0 K 2024-11-24T16:40:57,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/1217d6c3992d469ba62719d334db9209 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1217d6c3992d469ba62719d334db9209 2024-11-24T16:40:57,357 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1217d6c3992d469ba62719d334db9209, entries=150, sequenceid=672, filesize=12.0 K 2024-11-24T16:40:57,358 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=6.71 KB/6870 for 950b8a656f36ed921dc512c3b64e83ba in 888ms, sequenceid=672, compaction requested=false 2024-11-24T16:40:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 950b8a656f36ed921dc512c3b64e83ba: 2024-11-24T16:40:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:40:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-24T16:40:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-24T16:40:57,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T16:40:57,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9870 sec 2024-11-24T16:40:57,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.9940 sec 2024-11-24T16:40:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T16:40:57,514 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:40:57,985 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-24T16:40:57,985 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:57,985 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:57,985 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:57,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:59,335 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T16:40:59,855 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-24T16:40:59,855 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:59,855 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,855 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:59,886 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-24T16:40:59,887 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:59,887 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,887 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:59,916 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-24T16:40:59,916 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:40:59,916 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,917 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 198 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 225 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2278 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2217 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T16:40:59,917 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1020 2024-11-24T16:40:59,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3056 rows 2024-11-24T16:40:59,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1022 2024-11-24T16:40:59,918 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3062 rows 2024-11-24T16:40:59,918 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:40:59,918 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:40:59,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:40:59,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:40:59,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T16:40:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T16:40:59,936 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466459936"}]},"ts":"1732466459936"} 2024-11-24T16:40:59,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T16:40:59,938 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T16:40:59,938 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T16:40:59,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:40:59,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, UNASSIGN}] 2024-11-24T16:40:59,950 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, UNASSIGN 2024-11-24T16:40:59,952 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=950b8a656f36ed921dc512c3b64e83ba, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:40:59,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, UNASSIGN because future has completed 2024-11-24T16:40:59,960 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:40:59,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T16:41:00,120 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:41:00,120 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:41:00,120 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 950b8a656f36ed921dc512c3b64e83ba, disabling compactions & flushes 2024-11-24T16:41:00,121 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. after waiting 0 ms 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:41:00,121 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 950b8a656f36ed921dc512c3b64e83ba 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=A 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=B 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 950b8a656f36ed921dc512c3b64e83ba, store=C 2024-11-24T16:41:00,121 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:00,125 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/82efa0c6944646258ff710b9989c2add is 50, key is test_row_1/A:col10/1732466459914/Put/seqid=0 2024-11-24T16:41:00,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741954_1130 (size=9857) 2024-11-24T16:41:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T16:41:00,530 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=681 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/82efa0c6944646258ff710b9989c2add 2024-11-24T16:41:00,538 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/49aca09b49404f86ba0cab52f7452a0f is 50, key is test_row_1/B:col10/1732466459914/Put/seqid=0 2024-11-24T16:41:00,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741955_1131 (size=9857) 2024-11-24T16:41:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T16:41:00,942 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=681 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/49aca09b49404f86ba0cab52f7452a0f 2024-11-24T16:41:00,956 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/4f45acfb32b14db5aa21ac715db90b95 is 50, key is test_row_1/C:col10/1732466459914/Put/seqid=0 2024-11-24T16:41:00,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741956_1132 (size=9857) 2024-11-24T16:41:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T16:41:01,362 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=681 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/4f45acfb32b14db5aa21ac715db90b95 2024-11-24T16:41:01,371 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/A/82efa0c6944646258ff710b9989c2add as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/82efa0c6944646258ff710b9989c2add 2024-11-24T16:41:01,377 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/82efa0c6944646258ff710b9989c2add, entries=100, sequenceid=681, filesize=9.6 K 2024-11-24T16:41:01,379 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/B/49aca09b49404f86ba0cab52f7452a0f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/49aca09b49404f86ba0cab52f7452a0f 2024-11-24T16:41:01,384 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/49aca09b49404f86ba0cab52f7452a0f, entries=100, sequenceid=681, filesize=9.6 K 2024-11-24T16:41:01,385 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/.tmp/C/4f45acfb32b14db5aa21ac715db90b95 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/4f45acfb32b14db5aa21ac715db90b95 2024-11-24T16:41:01,397 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/4f45acfb32b14db5aa21ac715db90b95, entries=100, sequenceid=681, filesize=9.6 K 2024-11-24T16:41:01,398 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 950b8a656f36ed921dc512c3b64e83ba in 1276ms, sequenceid=681, compaction requested=true 2024-11-24T16:41:01,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/128c800074ed48be8f53c6705975f56b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/c6ce6f29f62d47a6a1d0c9bf4b53de44, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/45c3c5a501d84a558bcbd331459ebe79, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a92c14ad15184497883c6b5090e73d1c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1e84e2019e574773a188a9e660542347, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/e5869c71d1b3434f9bc840df4d023a16, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/0a246fcd1baa40d981526f01b4f7899f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/7d9e01a6f8984e90b1a9c7f4b19b5bce, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4adc0ee67ed746a4ae28930abc51e562, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/ebf8d685cb9c4c9da791811a422ea64e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/870f8bddc7cd41aa922e0b90f7bb0dad, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5ee19f05eba941d9ba2137eff1e558f7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/42b41c7f91fe459dbbdf0dc048ed4a11, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6cc58532608542539a9f4249a9cb9d23, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/774e0c7984e54e3d80e51a56353fca5f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/26988a3c72004e09bfe09fe8c11156f5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/9020fc6452004d20812baf8df60c5928, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1cd51bbcd7be459dabdbc5fd5853c1fd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5c2083e2f8ff4b448e0cca60450bf28a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eba52c95704f41119c2264e5d4dfb78e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/2b56a53acfd54d29b33f517b8f37e050, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4cb55f2532864f12b346ee9e67bc46c4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/3dbea45ffc544433b5344f11a73c8484, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/302b50ed5ccf46c19afa424afcc902cd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/54057cef397e4512a62689732efc4eed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/96654314ba3f4412a20e2ad92aba8718, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d8475eb0a8e2477cb94ebe380ed27690, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/64c62d98ff154b34bf27965c6b5c55fe, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/925ddd4ce381414baec9df663e9b252b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/22e0c9598d524afc843d6a9caa96c040, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/b19caab6ecbd47a9b3d0a0b3553774d1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d5d6329e12c44fc9a2d1398464892b9a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4a4aeda7dcce4623aa3910615b98e9a2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6c804c7b7afc451e8e824294004d6f35, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eeea36a647cb4c9e8f5db0385be5bf23, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a9e3b1ead2504c4c89fee71f5489e7a2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/85a8da445bfd4e47b80bb4c917e70824] to archive 2024-11-24T16:41:01,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:01,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/128c800074ed48be8f53c6705975f56b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/128c800074ed48be8f53c6705975f56b 2024-11-24T16:41:01,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/c6ce6f29f62d47a6a1d0c9bf4b53de44 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/c6ce6f29f62d47a6a1d0c9bf4b53de44 2024-11-24T16:41:01,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/45c3c5a501d84a558bcbd331459ebe79 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/45c3c5a501d84a558bcbd331459ebe79 2024-11-24T16:41:01,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a92c14ad15184497883c6b5090e73d1c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a92c14ad15184497883c6b5090e73d1c 2024-11-24T16:41:01,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1e84e2019e574773a188a9e660542347 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1e84e2019e574773a188a9e660542347 2024-11-24T16:41:01,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/e5869c71d1b3434f9bc840df4d023a16 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/e5869c71d1b3434f9bc840df4d023a16 2024-11-24T16:41:01,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/0a246fcd1baa40d981526f01b4f7899f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/0a246fcd1baa40d981526f01b4f7899f 2024-11-24T16:41:01,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/7d9e01a6f8984e90b1a9c7f4b19b5bce to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/7d9e01a6f8984e90b1a9c7f4b19b5bce 2024-11-24T16:41:01,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4adc0ee67ed746a4ae28930abc51e562 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4adc0ee67ed746a4ae28930abc51e562 2024-11-24T16:41:01,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/ebf8d685cb9c4c9da791811a422ea64e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/ebf8d685cb9c4c9da791811a422ea64e 2024-11-24T16:41:01,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/870f8bddc7cd41aa922e0b90f7bb0dad to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/870f8bddc7cd41aa922e0b90f7bb0dad 2024-11-24T16:41:01,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5ee19f05eba941d9ba2137eff1e558f7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5ee19f05eba941d9ba2137eff1e558f7 2024-11-24T16:41:01,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/42b41c7f91fe459dbbdf0dc048ed4a11 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/42b41c7f91fe459dbbdf0dc048ed4a11 2024-11-24T16:41:01,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6cc58532608542539a9f4249a9cb9d23 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6cc58532608542539a9f4249a9cb9d23 2024-11-24T16:41:01,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/774e0c7984e54e3d80e51a56353fca5f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/774e0c7984e54e3d80e51a56353fca5f 2024-11-24T16:41:01,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/26988a3c72004e09bfe09fe8c11156f5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/26988a3c72004e09bfe09fe8c11156f5 2024-11-24T16:41:01,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/9020fc6452004d20812baf8df60c5928 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/9020fc6452004d20812baf8df60c5928 2024-11-24T16:41:01,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1cd51bbcd7be459dabdbc5fd5853c1fd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/1cd51bbcd7be459dabdbc5fd5853c1fd 2024-11-24T16:41:01,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5c2083e2f8ff4b448e0cca60450bf28a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/5c2083e2f8ff4b448e0cca60450bf28a 2024-11-24T16:41:01,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eba52c95704f41119c2264e5d4dfb78e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eba52c95704f41119c2264e5d4dfb78e 2024-11-24T16:41:01,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/2b56a53acfd54d29b33f517b8f37e050 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/2b56a53acfd54d29b33f517b8f37e050 2024-11-24T16:41:01,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4cb55f2532864f12b346ee9e67bc46c4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4cb55f2532864f12b346ee9e67bc46c4 2024-11-24T16:41:01,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/3dbea45ffc544433b5344f11a73c8484 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/3dbea45ffc544433b5344f11a73c8484 2024-11-24T16:41:01,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/302b50ed5ccf46c19afa424afcc902cd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/302b50ed5ccf46c19afa424afcc902cd 2024-11-24T16:41:01,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/54057cef397e4512a62689732efc4eed to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/54057cef397e4512a62689732efc4eed 2024-11-24T16:41:01,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/96654314ba3f4412a20e2ad92aba8718 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/96654314ba3f4412a20e2ad92aba8718 2024-11-24T16:41:01,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d8475eb0a8e2477cb94ebe380ed27690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d8475eb0a8e2477cb94ebe380ed27690 2024-11-24T16:41:01,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/64c62d98ff154b34bf27965c6b5c55fe to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/64c62d98ff154b34bf27965c6b5c55fe 2024-11-24T16:41:01,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/925ddd4ce381414baec9df663e9b252b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/925ddd4ce381414baec9df663e9b252b 2024-11-24T16:41:01,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/22e0c9598d524afc843d6a9caa96c040 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/22e0c9598d524afc843d6a9caa96c040 2024-11-24T16:41:01,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/b19caab6ecbd47a9b3d0a0b3553774d1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/b19caab6ecbd47a9b3d0a0b3553774d1 2024-11-24T16:41:01,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d5d6329e12c44fc9a2d1398464892b9a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/d5d6329e12c44fc9a2d1398464892b9a 2024-11-24T16:41:01,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4a4aeda7dcce4623aa3910615b98e9a2 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/4a4aeda7dcce4623aa3910615b98e9a2 2024-11-24T16:41:01,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6c804c7b7afc451e8e824294004d6f35 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/6c804c7b7afc451e8e824294004d6f35 2024-11-24T16:41:01,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eeea36a647cb4c9e8f5db0385be5bf23 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/eeea36a647cb4c9e8f5db0385be5bf23 2024-11-24T16:41:01,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a9e3b1ead2504c4c89fee71f5489e7a2 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/a9e3b1ead2504c4c89fee71f5489e7a2 2024-11-24T16:41:01,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/85a8da445bfd4e47b80bb4c917e70824 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/85a8da445bfd4e47b80bb4c917e70824 2024-11-24T16:41:01,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0d43a86dd067494281bc38b011e92495, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/bca270ac0e99453d84dd0714b8746e49, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/f20019b2e9e0494fbe1e7ea712f642cc, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cde3f8851544467389a5e63e11b67d4f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ea07ff56b23245dc85308dd4bdc58e9e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd09775f227143c1b1a00e6c52e620a4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1830371470c84243904944007f502217, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0206976fa0674ec48a8cdc0e22978839, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2355539ba80a42a984bb6c46e930ad4a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fb6da2eefd3b4388999c7f133c042d1b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/54af325854f74dcbbb790e20e92f1800, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2f5446ee6f014993b2d515a6b5896244, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/d69e451653b54a5a8c62c7c7cc070510, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/180f6604e3de4964b152adb0d56f6d4b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fabfda338f4b4eb889d585a2de7754cb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/751864c62b634adea1908f9d18308de3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b243e540f9e942df86f9494a48479d4c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/a36c1c4b2caf4599ac60f2a255e83675, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ec00ce162b7e4cfc8ce81954b97d0de5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc8c6b296f6e40e2925a01f3247ae316, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd7f30722da848f7a1254c29b74ac63d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/8337b95aa86d46ae8c6bb2870f377ab3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/95783e5d438342b5892ffc2380bb8a8b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1907077f78c84a45aed1296a3ec2d4be, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/c082a2d885094b83aae6b71eae152c99, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/3b4b15f9429b4b20b3fbb2a64ffcca15, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/774e3e526cfd4937966c1641c3e806ee, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b83a953a8ede498db62022513193be0b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1906e94f72a344aba43013f0f47c0879, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b1a9276e51bf4f0599b531aae230de45, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc914c1aaf0e45fe811d9612b1969ebf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/16c57baefbcd4f8b9d41a6c35bfd27df, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/adf4a81d5a504675a5fd511cf8ec1b7c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/683a55884d12444cae26bbc9cc79701d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/acca177596524c71bd47744e0eaf7786, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/01c3b220df654561b065d285fc7a46ef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/62083fc2646d4b4bbaa3a2221bae85c8] to archive 2024-11-24T16:41:01,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:01,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0d43a86dd067494281bc38b011e92495 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0d43a86dd067494281bc38b011e92495 2024-11-24T16:41:01,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/bca270ac0e99453d84dd0714b8746e49 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/bca270ac0e99453d84dd0714b8746e49 2024-11-24T16:41:01,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/f20019b2e9e0494fbe1e7ea712f642cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/f20019b2e9e0494fbe1e7ea712f642cc 2024-11-24T16:41:01,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cde3f8851544467389a5e63e11b67d4f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cde3f8851544467389a5e63e11b67d4f 2024-11-24T16:41:01,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ea07ff56b23245dc85308dd4bdc58e9e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ea07ff56b23245dc85308dd4bdc58e9e 2024-11-24T16:41:01,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd09775f227143c1b1a00e6c52e620a4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd09775f227143c1b1a00e6c52e620a4 2024-11-24T16:41:01,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1830371470c84243904944007f502217 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1830371470c84243904944007f502217 2024-11-24T16:41:01,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0206976fa0674ec48a8cdc0e22978839 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/0206976fa0674ec48a8cdc0e22978839 2024-11-24T16:41:01,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2355539ba80a42a984bb6c46e930ad4a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2355539ba80a42a984bb6c46e930ad4a 2024-11-24T16:41:01,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fb6da2eefd3b4388999c7f133c042d1b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fb6da2eefd3b4388999c7f133c042d1b 2024-11-24T16:41:01,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/54af325854f74dcbbb790e20e92f1800 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/54af325854f74dcbbb790e20e92f1800 2024-11-24T16:41:01,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2f5446ee6f014993b2d515a6b5896244 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/2f5446ee6f014993b2d515a6b5896244 2024-11-24T16:41:01,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/d69e451653b54a5a8c62c7c7cc070510 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/d69e451653b54a5a8c62c7c7cc070510 2024-11-24T16:41:01,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/180f6604e3de4964b152adb0d56f6d4b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/180f6604e3de4964b152adb0d56f6d4b 2024-11-24T16:41:01,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fabfda338f4b4eb889d585a2de7754cb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fabfda338f4b4eb889d585a2de7754cb 2024-11-24T16:41:01,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/751864c62b634adea1908f9d18308de3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/751864c62b634adea1908f9d18308de3 2024-11-24T16:41:01,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b243e540f9e942df86f9494a48479d4c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b243e540f9e942df86f9494a48479d4c 2024-11-24T16:41:01,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/a36c1c4b2caf4599ac60f2a255e83675 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/a36c1c4b2caf4599ac60f2a255e83675 2024-11-24T16:41:01,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ec00ce162b7e4cfc8ce81954b97d0de5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/ec00ce162b7e4cfc8ce81954b97d0de5 2024-11-24T16:41:01,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc8c6b296f6e40e2925a01f3247ae316 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc8c6b296f6e40e2925a01f3247ae316 2024-11-24T16:41:01,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd7f30722da848f7a1254c29b74ac63d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/fd7f30722da848f7a1254c29b74ac63d 2024-11-24T16:41:01,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/8337b95aa86d46ae8c6bb2870f377ab3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/8337b95aa86d46ae8c6bb2870f377ab3 2024-11-24T16:41:01,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/95783e5d438342b5892ffc2380bb8a8b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/95783e5d438342b5892ffc2380bb8a8b 2024-11-24T16:41:01,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1907077f78c84a45aed1296a3ec2d4be to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1907077f78c84a45aed1296a3ec2d4be 2024-11-24T16:41:01,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/c082a2d885094b83aae6b71eae152c99 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/c082a2d885094b83aae6b71eae152c99 2024-11-24T16:41:01,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/3b4b15f9429b4b20b3fbb2a64ffcca15 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/3b4b15f9429b4b20b3fbb2a64ffcca15 2024-11-24T16:41:01,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/774e3e526cfd4937966c1641c3e806ee to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/774e3e526cfd4937966c1641c3e806ee 2024-11-24T16:41:01,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b83a953a8ede498db62022513193be0b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b83a953a8ede498db62022513193be0b 2024-11-24T16:41:01,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1906e94f72a344aba43013f0f47c0879 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/1906e94f72a344aba43013f0f47c0879 2024-11-24T16:41:01,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b1a9276e51bf4f0599b531aae230de45 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/b1a9276e51bf4f0599b531aae230de45 2024-11-24T16:41:01,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc914c1aaf0e45fe811d9612b1969ebf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/cc914c1aaf0e45fe811d9612b1969ebf 2024-11-24T16:41:01,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/16c57baefbcd4f8b9d41a6c35bfd27df to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/16c57baefbcd4f8b9d41a6c35bfd27df 2024-11-24T16:41:01,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/adf4a81d5a504675a5fd511cf8ec1b7c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/adf4a81d5a504675a5fd511cf8ec1b7c 2024-11-24T16:41:01,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/683a55884d12444cae26bbc9cc79701d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/683a55884d12444cae26bbc9cc79701d 2024-11-24T16:41:01,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/acca177596524c71bd47744e0eaf7786 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/acca177596524c71bd47744e0eaf7786 2024-11-24T16:41:01,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/01c3b220df654561b065d285fc7a46ef to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/01c3b220df654561b065d285fc7a46ef 2024-11-24T16:41:01,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/62083fc2646d4b4bbaa3a2221bae85c8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/62083fc2646d4b4bbaa3a2221bae85c8 2024-11-24T16:41:01,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a08247e63bb84c8da96bc0d593e21edf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/091b53ea0f2b4acbbd8a0293ebe7a0ef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35d835583c514c9781abf2b58884724c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d4a4ef61ba0d4b2ba654f464f3be2004, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35b630f3e7c44426b76253cf9ff6bb15, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f23efbc6c0fd4001b36f588f6e199460, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/da61c876c65641198734363ce871608d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/90d89da8e557495b86f55d48c773d7d8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6516c8ea61124af48c57949c22cd04fa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/509531cdbee0479dbb76661415d9e8ef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/e792238ecba24d7bb860bd0653618b85, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/db3f581d2d0940e8af31e80188d479dc, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a1e6b5368cb048b6a19043dee156ec97, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d10c557dcd74482f8a9a8866f0952e04, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1c3be3ab8cd046e59ddbe359c10dfc66, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3a80e7446d0b446095043dee021c01ef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/2d3bc21f6e774c40b39900047959b267, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3efb0b5903fc462580705fae608d21bd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3e4e3699ebc64ab68b8c25291c67b87e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d645fd7fadef4715a8e1a50e4fd746ae, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/dd8dc12de3904f4a93098d9a824e632f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/01be69f362fb40978496b4888e565df2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3474908dc8974db491450d3d35bf710e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/09eeda0c2a58466f875c74ed077a7f67, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3ceaa7b3969b4bf69bac110b014f6dd0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f9584e51b7a64da99551189fd6a688b1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/00afe6acf3294801a4bffa855bfd1df9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9aa0f1bedfd449d29adcd2632a203aae, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/bd4193b3713e4bc5a9c3eb0b1f369dd5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6470e1ea2707422a8da490bb147bc8df, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/904a961326c24f3b82002053c1a19937, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/5fc3521944044c16912fc3a1a8b72a31, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/80202f14cacd44f39b57ce9ba66a1da2] to archive 2024-11-24T16:41:01,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:01,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a08247e63bb84c8da96bc0d593e21edf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a08247e63bb84c8da96bc0d593e21edf 2024-11-24T16:41:01,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/091b53ea0f2b4acbbd8a0293ebe7a0ef to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/091b53ea0f2b4acbbd8a0293ebe7a0ef 2024-11-24T16:41:01,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35d835583c514c9781abf2b58884724c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35d835583c514c9781abf2b58884724c 2024-11-24T16:41:01,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d4a4ef61ba0d4b2ba654f464f3be2004 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d4a4ef61ba0d4b2ba654f464f3be2004 2024-11-24T16:41:01,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35b630f3e7c44426b76253cf9ff6bb15 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/35b630f3e7c44426b76253cf9ff6bb15 2024-11-24T16:41:01,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f23efbc6c0fd4001b36f588f6e199460 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f23efbc6c0fd4001b36f588f6e199460 2024-11-24T16:41:01,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/da61c876c65641198734363ce871608d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/da61c876c65641198734363ce871608d 2024-11-24T16:41:01,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/90d89da8e557495b86f55d48c773d7d8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/90d89da8e557495b86f55d48c773d7d8 2024-11-24T16:41:01,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6516c8ea61124af48c57949c22cd04fa to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6516c8ea61124af48c57949c22cd04fa 2024-11-24T16:41:01,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/509531cdbee0479dbb76661415d9e8ef to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/509531cdbee0479dbb76661415d9e8ef 2024-11-24T16:41:01,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/e792238ecba24d7bb860bd0653618b85 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/e792238ecba24d7bb860bd0653618b85 2024-11-24T16:41:01,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/db3f581d2d0940e8af31e80188d479dc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/db3f581d2d0940e8af31e80188d479dc 2024-11-24T16:41:01,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a1e6b5368cb048b6a19043dee156ec97 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/a1e6b5368cb048b6a19043dee156ec97 2024-11-24T16:41:01,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/ad9b0d25f4f84a8eb26c3163e6b2f5f7 2024-11-24T16:41:01,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d10c557dcd74482f8a9a8866f0952e04 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d10c557dcd74482f8a9a8866f0952e04 2024-11-24T16:41:01,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1c3be3ab8cd046e59ddbe359c10dfc66 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1c3be3ab8cd046e59ddbe359c10dfc66 2024-11-24T16:41:01,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3a80e7446d0b446095043dee021c01ef to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3a80e7446d0b446095043dee021c01ef 2024-11-24T16:41:01,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/2d3bc21f6e774c40b39900047959b267 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/2d3bc21f6e774c40b39900047959b267 2024-11-24T16:41:01,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3efb0b5903fc462580705fae608d21bd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3efb0b5903fc462580705fae608d21bd 2024-11-24T16:41:01,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3e4e3699ebc64ab68b8c25291c67b87e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3e4e3699ebc64ab68b8c25291c67b87e 2024-11-24T16:41:01,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d645fd7fadef4715a8e1a50e4fd746ae to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d645fd7fadef4715a8e1a50e4fd746ae 2024-11-24T16:41:01,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/dd8dc12de3904f4a93098d9a824e632f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/dd8dc12de3904f4a93098d9a824e632f 2024-11-24T16:41:01,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/01be69f362fb40978496b4888e565df2 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/01be69f362fb40978496b4888e565df2 2024-11-24T16:41:01,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3474908dc8974db491450d3d35bf710e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3474908dc8974db491450d3d35bf710e 2024-11-24T16:41:01,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/d7feb33b1e7d4b0fb8a85bd3eeeddbc8 2024-11-24T16:41:01,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/09eeda0c2a58466f875c74ed077a7f67 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/09eeda0c2a58466f875c74ed077a7f67 2024-11-24T16:41:01,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9f60f4fad65f41bfb7a5ac5aa2d2d0ad 2024-11-24T16:41:01,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3ceaa7b3969b4bf69bac110b014f6dd0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/3ceaa7b3969b4bf69bac110b014f6dd0 2024-11-24T16:41:01,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f9584e51b7a64da99551189fd6a688b1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/f9584e51b7a64da99551189fd6a688b1 2024-11-24T16:41:01,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/00afe6acf3294801a4bffa855bfd1df9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/00afe6acf3294801a4bffa855bfd1df9 2024-11-24T16:41:01,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9aa0f1bedfd449d29adcd2632a203aae to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/9aa0f1bedfd449d29adcd2632a203aae 2024-11-24T16:41:01,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/95f5c95f2fbe4a50a3e7f4aa770b6ef9 2024-11-24T16:41:01,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/bd4193b3713e4bc5a9c3eb0b1f369dd5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/bd4193b3713e4bc5a9c3eb0b1f369dd5 2024-11-24T16:41:01,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6470e1ea2707422a8da490bb147bc8df to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/6470e1ea2707422a8da490bb147bc8df 2024-11-24T16:41:01,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/904a961326c24f3b82002053c1a19937 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/904a961326c24f3b82002053c1a19937 2024-11-24T16:41:01,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/5fc3521944044c16912fc3a1a8b72a31 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/5fc3521944044c16912fc3a1a8b72a31 2024-11-24T16:41:01,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/80202f14cacd44f39b57ce9ba66a1da2 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/80202f14cacd44f39b57ce9ba66a1da2 2024-11-24T16:41:01,645 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/recovered.edits/684.seqid, newMaxSeqId=684, maxSeqId=1 2024-11-24T16:41:01,646 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. 2024-11-24T16:41:01,646 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 950b8a656f36ed921dc512c3b64e83ba: Waiting for close lock at 1732466460120Running coprocessor pre-close hooks at 1732466460120Disabling compacts and flushes for region at 1732466460120Disabling writes for close at 1732466460121 (+1 ms)Obtaining lock to block concurrent updates at 1732466460121Preparing flush snapshotting stores in 950b8a656f36ed921dc512c3b64e83ba at 1732466460121Finished memstore snapshotting TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732466460121Flushing stores of TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba. at 1732466460122 (+1 ms)Flushing 950b8a656f36ed921dc512c3b64e83ba/A: creating writer at 1732466460122Flushing 950b8a656f36ed921dc512c3b64e83ba/A: appending metadata at 1732466460125 (+3 ms)Flushing 950b8a656f36ed921dc512c3b64e83ba/A: closing flushed file at 1732466460125Flushing 950b8a656f36ed921dc512c3b64e83ba/B: creating writer at 1732466460535 (+410 ms)Flushing 950b8a656f36ed921dc512c3b64e83ba/B: appending metadata at 1732466460537 (+2 ms)Flushing 950b8a656f36ed921dc512c3b64e83ba/B: closing flushed file at 1732466460537Flushing 950b8a656f36ed921dc512c3b64e83ba/C: creating writer at 1732466460952 (+415 ms)Flushing 950b8a656f36ed921dc512c3b64e83ba/C: appending metadata at 1732466460955 (+3 ms)Flushing 950b8a656f36ed921dc512c3b64e83ba/C: closing flushed file at 1732466460955Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d74c489: reopening flushed file at 1732466461369 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dbfaaf9: reopening flushed file at 1732466461377 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@774d3eef: reopening flushed file at 1732466461385 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 950b8a656f36ed921dc512c3b64e83ba in 1276ms, sequenceid=681, compaction requested=true at 1732466461398 (+13 ms)Writing region close event to WAL at 1732466461641 (+243 ms)Running coprocessor post-close hooks at 1732466461645 (+4 ms)Closed at 1732466461646 (+1 ms) 2024-11-24T16:41:01,649 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:41:01,649 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=950b8a656f36ed921dc512c3b64e83ba, regionState=CLOSED 2024-11-24T16:41:01,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:01,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T16:41:01,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 950b8a656f36ed921dc512c3b64e83ba, server=e335232f7491,33607,1732466431346 in 1.6920 sec 2024-11-24T16:41:01,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T16:41:01,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=950b8a656f36ed921dc512c3b64e83ba, UNASSIGN in 1.7070 sec 2024-11-24T16:41:01,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-24T16:41:01,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7160 sec 2024-11-24T16:41:01,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466461659"}]},"ts":"1732466461659"} 2024-11-24T16:41:01,661 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T16:41:01,661 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T16:41:01,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7350 sec 2024-11-24T16:41:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T16:41:02,085 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:02,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T16:41:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:02,097 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:02,099 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T16:41:02,120 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:41:02,132 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/recovered.edits] 2024-11-24T16:41:02,138 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/82efa0c6944646258ff710b9989c2add to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/82efa0c6944646258ff710b9989c2add 2024-11-24T16:41:02,140 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/de4dcb68a35d480a867b8956459a3955 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/de4dcb68a35d480a867b8956459a3955 2024-11-24T16:41:02,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/efd8f2cbde1d471a8ff86cec986873ce to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/A/efd8f2cbde1d471a8ff86cec986873ce 2024-11-24T16:41:02,149 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/49aca09b49404f86ba0cab52f7452a0f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/49aca09b49404f86ba0cab52f7452a0f 2024-11-24T16:41:02,151 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/854915a04b5b4d509150dc1391b529bf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/854915a04b5b4d509150dc1391b529bf 2024-11-24T16:41:02,154 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/e8ed64e767a14adaad485afabe9eb845 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/B/e8ed64e767a14adaad485afabe9eb845 2024-11-24T16:41:02,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/0943fc5e09a94226af7b17aaa86e2f7f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/0943fc5e09a94226af7b17aaa86e2f7f 2024-11-24T16:41:02,169 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1217d6c3992d469ba62719d334db9209 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/1217d6c3992d469ba62719d334db9209 2024-11-24T16:41:02,171 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/4f45acfb32b14db5aa21ac715db90b95 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/C/4f45acfb32b14db5aa21ac715db90b95 2024-11-24T16:41:02,186 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/recovered.edits/684.seqid to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba/recovered.edits/684.seqid 2024-11-24T16:41:02,187 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/950b8a656f36ed921dc512c3b64e83ba 2024-11-24T16:41:02,187 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T16:41:02,194 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:02,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T16:41:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T16:41:02,210 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T16:41:02,276 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T16:41:02,278 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:02,278 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T16:41:02,279 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732466462279"}]},"ts":"9223372036854775807"} 2024-11-24T16:41:02,288 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T16:41:02,289 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 950b8a656f36ed921dc512c3b64e83ba, NAME => 'TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T16:41:02,289 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T16:41:02,289 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732466462289"}]},"ts":"9223372036854775807"} 2024-11-24T16:41:02,293 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T16:41:02,295 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:02,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 204 msec 2024-11-24T16:41:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T16:41:02,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T16:41:02,415 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:02,429 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=245 (was 209) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e335232f7491:33607-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e335232f7491:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2114816948_22 at /127.0.0.1:36142 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e335232f7491:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 315) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6516 (was 7113) 2024-11-24T16:41:02,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=245, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=6516 2024-11-24T16:41:02,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:41:02,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:41:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:02,445 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:41:02,446 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:02,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-24T16:41:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T16:41:02,447 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:41:02,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741957_1133 (size=1061) 2024-11-24T16:41:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T16:41:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T16:41:02,858 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:41:02,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741958_1134 (size=53) 2024-11-24T16:41:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T16:41:03,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:41:03,265 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 68db56cd5236f933146f262f370fc8cc, disabling compactions & flushes 2024-11-24T16:41:03,265 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,265 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,265 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. after waiting 0 ms 2024-11-24T16:41:03,265 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,265 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,265 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 68db56cd5236f933146f262f370fc8cc: Waiting for close lock at 1732466463265Disabling compacts and flushes for region at 1732466463265Disabling writes for close at 1732466463265Writing region close event to WAL at 1732466463265Closed at 1732466463265 2024-11-24T16:41:03,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:41:03,267 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732466463267"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466463267"}]},"ts":"1732466463267"} 2024-11-24T16:41:03,269 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:41:03,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:41:03,271 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466463271"}]},"ts":"1732466463271"} 2024-11-24T16:41:03,273 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T16:41:03,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, ASSIGN}] 2024-11-24T16:41:03,274 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, ASSIGN 2024-11-24T16:41:03,275 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, ASSIGN; state=OFFLINE, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=false 2024-11-24T16:41:03,427 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:03,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, ASSIGN because future has completed 2024-11-24T16:41:03,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T16:41:03,586 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,587 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:41:03,587 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,587 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:41:03,587 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,588 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,589 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,591 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:03,592 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68db56cd5236f933146f262f370fc8cc columnFamilyName A 2024-11-24T16:41:03,592 DEBUG [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:03,593 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(327): Store=68db56cd5236f933146f262f370fc8cc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:03,593 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,595 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:03,595 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68db56cd5236f933146f262f370fc8cc columnFamilyName B 2024-11-24T16:41:03,595 DEBUG [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:03,596 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(327): Store=68db56cd5236f933146f262f370fc8cc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:03,596 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,597 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:03,597 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68db56cd5236f933146f262f370fc8cc columnFamilyName C 2024-11-24T16:41:03,597 DEBUG [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:03,598 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(327): Store=68db56cd5236f933146f262f370fc8cc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:03,598 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,598 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,599 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,599 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,600 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,600 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,601 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:41:03,602 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,604 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:41:03,605 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 68db56cd5236f933146f262f370fc8cc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69834066, jitterRate=0.04060867428779602}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:41:03,605 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:03,606 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 68db56cd5236f933146f262f370fc8cc: Running coprocessor pre-open hook at 1732466463588Writing region info on filesystem at 1732466463588Initializing all the Stores at 1732466463589 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466463589Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466463589Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466463589Cleaning up temporary data from old regions at 1732466463600 (+11 ms)Running coprocessor post-open hooks at 1732466463605 (+5 ms)Region opened successfully at 1732466463606 (+1 ms) 2024-11-24T16:41:03,647 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., pid=36, masterSystemTime=1732466463582 2024-11-24T16:41:03,650 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,650 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:03,651 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:03,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:03,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-24T16:41:03,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 in 223 msec 2024-11-24T16:41:03,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T16:41:03,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, ASSIGN in 382 msec 2024-11-24T16:41:03,659 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:41:03,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466463659"}]},"ts":"1732466463659"} 2024-11-24T16:41:03,660 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T16:41:03,661 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:41:03,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2180 sec 2024-11-24T16:41:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T16:41:04,594 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:04,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@244f1d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:04,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:04,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:04,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297cb622, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:04,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:04,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:04,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:04,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12afde2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:04,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:04,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:04,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:04,604 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:41:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,37979,1732466430336 2024-11-24T16:41:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@302feb9a 2024-11-24T16:41:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:41:04,605 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:41:04,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:41:04,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:41:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:04,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741959_1135 (size=1097) 2024-11-24T16:41:05,042 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T16:41:05,042 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T16:41:05,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:41:05,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, REOPEN/MOVE}] 2024-11-24T16:41:05,053 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, REOPEN/MOVE 2024-11-24T16:41:05,054 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, REOPEN/MOVE because future has completed 2024-11-24T16:41:05,056 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:41:05,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:05,208 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,208 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:41:05,208 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 68db56cd5236f933146f262f370fc8cc, disabling compactions & flushes 2024-11-24T16:41:05,208 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,208 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,208 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. after waiting 0 ms 2024-11-24T16:41:05,208 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,212 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T16:41:05,213 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,213 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 68db56cd5236f933146f262f370fc8cc: Waiting for close lock at 1732466465208Running coprocessor pre-close hooks at 1732466465208Disabling compacts and flushes for region at 1732466465208Disabling writes for close at 1732466465208Writing region close event to WAL at 1732466465209 (+1 ms)Running coprocessor post-close hooks at 1732466465213 (+4 ms)Closed at 1732466465213 2024-11-24T16:41:05,213 WARN [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 68db56cd5236f933146f262f370fc8cc to self. 2024-11-24T16:41:05,216 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,216 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=CLOSED 2024-11-24T16:41:05,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:05,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-24T16:41:05,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 in 163 msec 2024-11-24T16:41:05,222 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, REOPEN/MOVE; state=CLOSED, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=true 2024-11-24T16:41:05,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, REOPEN/MOVE because future has completed 2024-11-24T16:41:05,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:05,532 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,532 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:41:05,533 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,533 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:41:05,533 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,533 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,535 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,536 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:05,542 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68db56cd5236f933146f262f370fc8cc columnFamilyName A 2024-11-24T16:41:05,545 DEBUG [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:05,545 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(327): Store=68db56cd5236f933146f262f370fc8cc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:05,546 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,547 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:05,547 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68db56cd5236f933146f262f370fc8cc columnFamilyName B 2024-11-24T16:41:05,548 DEBUG [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:05,549 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(327): Store=68db56cd5236f933146f262f370fc8cc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:05,549 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,550 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:05,550 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 68db56cd5236f933146f262f370fc8cc columnFamilyName C 2024-11-24T16:41:05,550 DEBUG [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:05,552 INFO [StoreOpener-68db56cd5236f933146f262f370fc8cc-1 {}] regionserver.HStore(327): Store=68db56cd5236f933146f262f370fc8cc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:05,552 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,552 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,553 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,555 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,557 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,557 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,558 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:41:05,561 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,562 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 68db56cd5236f933146f262f370fc8cc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66756411, jitterRate=-0.005251958966255188}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:41:05,562 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,565 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 68db56cd5236f933146f262f370fc8cc: Running coprocessor pre-open hook at 1732466465533Writing region info on filesystem at 1732466465533Initializing all the Stores at 1732466465534 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466465534Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466465534Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466465534Cleaning up temporary data from old regions at 1732466465557 (+23 ms)Running coprocessor post-open hooks at 1732466465562 (+5 ms)Region opened successfully at 1732466465565 (+3 ms) 2024-11-24T16:41:05,566 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., pid=41, masterSystemTime=1732466465528 2024-11-24T16:41:05,570 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,570 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,570 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=OPEN, openSeqNum=5, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,573 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:05,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-24T16:41:05,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 in 198 msec 2024-11-24T16:41:05,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-24T16:41:05,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, REOPEN/MOVE in 533 msec 2024-11-24T16:41:05,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-24T16:41:05,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 544 msec 2024-11-24T16:41:05,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 977 msec 2024-11-24T16:41:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-24T16:41:05,596 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:05,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b4ffc72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,599 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1e09e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3520d4f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,605 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13aa3d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,607 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dae6dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,614 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c18104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62812d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,626 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e8fca4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8634d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,634 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621b033e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,637 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c12a8f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,670 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce63009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@feb3aa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,683 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1fa278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0f42b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829f175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51798ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:05,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:05,690 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:05,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540c1af1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:05,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:05,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:05,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7942fe6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:05,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:05,697 DEBUG [Thread-687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,697 DEBUG [Thread-685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,697 DEBUG [Thread-683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,697 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-24T16:41:05,699 DEBUG [Thread-680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,699 DEBUG [Thread-683 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,699 DEBUG [Thread-683 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,700 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,700 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,701 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:05,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:05,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T16:41:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,717 DEBUG [Thread-678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,718 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,720 DEBUG [Thread-676 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,720 DEBUG [Thread-676 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,723 DEBUG [Thread-689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:05,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:05,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:05,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:05,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:41:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:41:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:05,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732466465755, max timestamp=1732466465786Num uniques 859; 2024-11-24T16:41:05,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-24T16:41:05,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732466465755, max timestamp=1732466465786Num uniques 859; 2024-11-24T16:41:05,797 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-24T16:41:05,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241a656e5637b44a4ba698c9af811e2223_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466465726/Put/seqid=0 2024-11-24T16:41:05,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=509, cellSize=23305, totalHeapSize=53176, min timestamp=1732466465755, max timestamp=1732466465786Num uniques 859; 2024-11-24T16:41:05,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=23305, suffix heap size=117064,new segment heap size=53176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=509 2024-11-24T16:41:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T16:41:05,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741960_1136 (size=21918) 2024-11-24T16:41:05,833 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:05,846 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241a656e5637b44a4ba698c9af811e2223_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241a656e5637b44a4ba698c9af811e2223_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:05,847 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/56217b40019a4cc9a65e4abfcbca1dd4, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:05,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/56217b40019a4cc9a65e4abfcbca1dd4 is 175, key is test_row_0/A:col10/1732466465726/Put/seqid=0 2024-11-24T16:41:05,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:05,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:05,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:05,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741961_1137 (size=65323) 2024-11-24T16:41:05,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/56217b40019a4cc9a65e4abfcbca1dd4 2024-11-24T16:41:05,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466525863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466525866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:05,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466525868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466525871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:05,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466525873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:05,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:05,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/149e1b8e4bd74751be8d82d61a15420d is 50, key is test_row_0/B:col10/1732466465726/Put/seqid=0 2024-11-24T16:41:05,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741962_1138 (size=12001) 2024-11-24T16:41:06,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:06,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466525994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466525994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T16:41:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466525994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466525994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466526004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466526234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466526234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466526234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466526234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466526234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:06,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:06,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T16:41:06,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/149e1b8e4bd74751be8d82d61a15420d 2024-11-24T16:41:06,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/194806cf526b448a9d9f7cb3a6e0cd2c is 50, key is test_row_0/C:col10/1732466465726/Put/seqid=0 2024-11-24T16:41:06,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:06,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:06,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741963_1139 (size=12001) 2024-11-24T16:41:06,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/194806cf526b448a9d9f7cb3a6e0cd2c 2024-11-24T16:41:06,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/56217b40019a4cc9a65e4abfcbca1dd4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/56217b40019a4cc9a65e4abfcbca1dd4 2024-11-24T16:41:06,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/56217b40019a4cc9a65e4abfcbca1dd4, entries=350, sequenceid=15, filesize=63.8 K 2024-11-24T16:41:06,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/149e1b8e4bd74751be8d82d61a15420d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/149e1b8e4bd74751be8d82d61a15420d 2024-11-24T16:41:06,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/149e1b8e4bd74751be8d82d61a15420d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T16:41:06,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/194806cf526b448a9d9f7cb3a6e0cd2c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/194806cf526b448a9d9f7cb3a6e0cd2c 2024-11-24T16:41:06,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/194806cf526b448a9d9f7cb3a6e0cd2c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T16:41:06,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 68db56cd5236f933146f262f370fc8cc in 827ms, sequenceid=15, compaction requested=false 2024-11-24T16:41:06,559 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T16:41:06,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:06,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=140.89 KB heapSize=372.09 KB 2024-11-24T16:41:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:06,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:06,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d94a99719cb14d2cb4d82233d1447ed4_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466466567/Put/seqid=0 2024-11-24T16:41:06,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466526593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466526593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:06,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466526602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741964_1140 (size=14594) 2024-11-24T16:41:06,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466526615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466526627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466526734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466526735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466526744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466526745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466526735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:06,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T16:41:06,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:06,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:06,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:06,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466526963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466526964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466526964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466526964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466526964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,035 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:07,043 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d94a99719cb14d2cb4d82233d1447ed4_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d94a99719cb14d2cb4d82233d1447ed4_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:07,044 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/e2d389fcf68a46bab6e5f6b47b8615c5, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:07,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/e2d389fcf68a46bab6e5f6b47b8615c5 is 175, key is test_row_0/A:col10/1732466466567/Put/seqid=0 2024-11-24T16:41:07,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741965_1141 (size=39549) 2024-11-24T16:41:07,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/e2d389fcf68a46bab6e5f6b47b8615c5 2024-11-24T16:41:07,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/e59789c06e65495587b6c2696fe9ef2a is 50, key is test_row_0/B:col10/1732466466567/Put/seqid=0 2024-11-24T16:41:07,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741966_1142 (size=12001) 2024-11-24T16:41:07,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/e59789c06e65495587b6c2696fe9ef2a 2024-11-24T16:41:07,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:07,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:07,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:07,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/1b4e04a755d844878265fce3d5a3fda5 is 50, key is test_row_0/C:col10/1732466466567/Put/seqid=0 2024-11-24T16:41:07,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741967_1143 (size=12001) 2024-11-24T16:41:07,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/1b4e04a755d844878265fce3d5a3fda5 2024-11-24T16:41:07,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/e2d389fcf68a46bab6e5f6b47b8615c5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/e2d389fcf68a46bab6e5f6b47b8615c5 2024-11-24T16:41:07,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/e2d389fcf68a46bab6e5f6b47b8615c5, entries=200, sequenceid=46, filesize=38.6 K 2024-11-24T16:41:07,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/e59789c06e65495587b6c2696fe9ef2a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/e59789c06e65495587b6c2696fe9ef2a 2024-11-24T16:41:07,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/e59789c06e65495587b6c2696fe9ef2a, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T16:41:07,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/1b4e04a755d844878265fce3d5a3fda5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1b4e04a755d844878265fce3d5a3fda5 2024-11-24T16:41:07,210 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:41:07,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1b4e04a755d844878265fce3d5a3fda5, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T16:41:07,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~372.05 KB/380976, currentSize=46.96 KB/48090 for 68db56cd5236f933146f262f370fc8cc in 646ms, sequenceid=46, compaction requested=false 2024-11-24T16:41:07,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:07,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T16:41:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:07,250 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:41:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:07,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:07,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:07,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:07,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:07,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:07,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ed9ffebdf9ed449e815cd993dd578d36_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466466586/Put/seqid=0 2024-11-24T16:41:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:07,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:07,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741968_1144 (size=12154) 2024-11-24T16:41:07,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466467325, max timestamp=1732466467355Num uniques 859; 2024-11-24T16:41:07,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:07,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466467325, max timestamp=1732466467355Num uniques 859; 2024-11-24T16:41:07,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:07,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466467325, max timestamp=1732466467355Num uniques 859; 2024-11-24T16:41:07,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:07,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466527425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466527426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466527435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466527437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466527439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466527544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466527554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466527555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466527555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466527565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:07,751 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ed9ffebdf9ed449e815cd993dd578d36_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ed9ffebdf9ed449e815cd993dd578d36_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:07,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/aa90d7e9a7eb4dd3bdabe2053ea5513b, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/aa90d7e9a7eb4dd3bdabe2053ea5513b is 175, key is test_row_0/A:col10/1732466466586/Put/seqid=0 2024-11-24T16:41:07,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466527755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466527764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466527764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466527764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:07,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466527773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:07,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741969_1145 (size=30955) 2024-11-24T16:41:07,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/aa90d7e9a7eb4dd3bdabe2053ea5513b 2024-11-24T16:41:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/bd7f00baa91d4ba793c2ac198d6c31b4 is 50, key is test_row_0/B:col10/1732466466586/Put/seqid=0 2024-11-24T16:41:07,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741970_1146 (size=12001) 2024-11-24T16:41:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T16:41:08,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466528074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466528074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466528074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466528074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466528094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,246 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/bd7f00baa91d4ba793c2ac198d6c31b4 2024-11-24T16:41:08,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/05dcaeab001745649cec779a6a020725 is 50, key is test_row_0/C:col10/1732466466586/Put/seqid=0 2024-11-24T16:41:08,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741971_1147 (size=12001) 2024-11-24T16:41:08,285 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/05dcaeab001745649cec779a6a020725 2024-11-24T16:41:08,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/aa90d7e9a7eb4dd3bdabe2053ea5513b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/aa90d7e9a7eb4dd3bdabe2053ea5513b 2024-11-24T16:41:08,300 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/aa90d7e9a7eb4dd3bdabe2053ea5513b, entries=150, sequenceid=56, filesize=30.2 K 2024-11-24T16:41:08,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/bd7f00baa91d4ba793c2ac198d6c31b4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/bd7f00baa91d4ba793c2ac198d6c31b4 2024-11-24T16:41:08,314 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/bd7f00baa91d4ba793c2ac198d6c31b4, entries=150, sequenceid=56, filesize=11.7 K 2024-11-24T16:41:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/05dcaeab001745649cec779a6a020725 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/05dcaeab001745649cec779a6a020725 2024-11-24T16:41:08,330 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/05dcaeab001745649cec779a6a020725, entries=150, sequenceid=56, filesize=11.7 K 2024-11-24T16:41:08,336 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 68db56cd5236f933146f262f370fc8cc in 1086ms, sequenceid=56, compaction requested=true 2024-11-24T16:41:08,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:08,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:08,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-24T16:41:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-24T16:41:08,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T16:41:08,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-11-24T16:41:08,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6450 sec 2024-11-24T16:41:08,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466467421, max timestamp=1732466468590Num uniques 1018; 2024-11-24T16:41:08,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:41:08,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466467421, max timestamp=1732466468590Num uniques 1018; 2024-11-24T16:41:08,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:41:08,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466467421, max timestamp=1732466468590Num uniques 1018; 2024-11-24T16:41:08,600 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:41:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:08,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-24T16:41:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:08,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:08,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241bff8d9394ab4b25806b96d3abe51c34_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466468601/Put/seqid=0 2024-11-24T16:41:08,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466468645, max timestamp=1732466468656Num uniques 859; 2024-11-24T16:41:08,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:41:08,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466468645, max timestamp=1732466468656Num uniques 859; 2024-11-24T16:41:08,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466468645, max timestamp=1732466468656Num uniques 859; 2024-11-24T16:41:08,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:41:08,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:41:08,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741972_1148 (size=14594) 2024-11-24T16:41:08,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:08,675 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241bff8d9394ab4b25806b96d3abe51c34_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241bff8d9394ab4b25806b96d3abe51c34_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:08,679 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/83677f7936d74c2ab493381e9055d4d7, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:08,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/83677f7936d74c2ab493381e9055d4d7 is 175, key is test_row_0/A:col10/1732466468601/Put/seqid=0 2024-11-24T16:41:08,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741973_1149 (size=39549) 2024-11-24T16:41:08,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/83677f7936d74c2ab493381e9055d4d7 2024-11-24T16:41:08,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/64ba858726224741826fa0d9c2d2f62f is 50, key is test_row_0/B:col10/1732466468601/Put/seqid=0 2024-11-24T16:41:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466528707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466528710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466528711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466528713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466528713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:08,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741974_1150 (size=12001) 2024-11-24T16:41:08,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/64ba858726224741826fa0d9c2d2f62f 2024-11-24T16:41:08,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/84d61fb2a23e41bf80f3f35ffac55d05 is 50, key is test_row_0/C:col10/1732466468601/Put/seqid=0 2024-11-24T16:41:08,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741975_1151 (size=12001) 2024-11-24T16:41:08,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/84d61fb2a23e41bf80f3f35ffac55d05 2024-11-24T16:41:08,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/83677f7936d74c2ab493381e9055d4d7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/83677f7936d74c2ab493381e9055d4d7 2024-11-24T16:41:08,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/83677f7936d74c2ab493381e9055d4d7, entries=200, sequenceid=98, filesize=38.6 K 2024-11-24T16:41:08,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/64ba858726224741826fa0d9c2d2f62f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/64ba858726224741826fa0d9c2d2f62f 2024-11-24T16:41:08,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/64ba858726224741826fa0d9c2d2f62f, entries=150, sequenceid=98, filesize=11.7 K 2024-11-24T16:41:08,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/84d61fb2a23e41bf80f3f35ffac55d05 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/84d61fb2a23e41bf80f3f35ffac55d05 2024-11-24T16:41:08,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/84d61fb2a23e41bf80f3f35ffac55d05, entries=150, sequenceid=98, filesize=11.7 K 2024-11-24T16:41:08,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=136.42 KB/139690 for 68db56cd5236f933146f262f370fc8cc in 209ms, sequenceid=98, compaction requested=true 2024-11-24T16:41:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:08,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:08,820 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:08,821 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:08,823 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:08,823 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/B is initiating minor compaction (all files) 2024-11-24T16:41:08,823 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/B in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:08,824 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/149e1b8e4bd74751be8d82d61a15420d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/e59789c06e65495587b6c2696fe9ef2a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/bd7f00baa91d4ba793c2ac198d6c31b4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/64ba858726224741826fa0d9c2d2f62f] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=46.9 K 2024-11-24T16:41:08,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:08,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:08,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:08,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:08,827 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 175376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:08,827 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/A is initiating minor compaction (all files) 2024-11-24T16:41:08,827 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/A in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:08,828 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/56217b40019a4cc9a65e4abfcbca1dd4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/e2d389fcf68a46bab6e5f6b47b8615c5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/aa90d7e9a7eb4dd3bdabe2053ea5513b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/83677f7936d74c2ab493381e9055d4d7] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=171.3 K 2024-11-24T16:41:08,828 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:08,828 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/56217b40019a4cc9a65e4abfcbca1dd4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/e2d389fcf68a46bab6e5f6b47b8615c5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/aa90d7e9a7eb4dd3bdabe2053ea5513b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/83677f7936d74c2ab493381e9055d4d7] 2024-11-24T16:41:08,829 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56217b40019a4cc9a65e4abfcbca1dd4, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732466465717 2024-11-24T16:41:08,829 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 149e1b8e4bd74751be8d82d61a15420d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732466465723 2024-11-24T16:41:08,829 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e59789c06e65495587b6c2696fe9ef2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732466465865 2024-11-24T16:41:08,830 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2d389fcf68a46bab6e5f6b47b8615c5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732466465830 2024-11-24T16:41:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:08,830 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting bd7f00baa91d4ba793c2ac198d6c31b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732466466586 2024-11-24T16:41:08,830 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa90d7e9a7eb4dd3bdabe2053ea5513b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732466466586 2024-11-24T16:41:08,831 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 64ba858726224741826fa0d9c2d2f62f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732466468593 2024-11-24T16:41:08,832 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83677f7936d74c2ab493381e9055d4d7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732466468593 2024-11-24T16:41:08,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=143.13 KB heapSize=405.53 KB 2024-11-24T16:41:08,835 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466468708, max timestamp=1732466468830Num uniques 1118; 2024-11-24T16:41:08,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-24T16:41:08,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466468708, max timestamp=1732466468831Num uniques 1068; 2024-11-24T16:41:08,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-24T16:41:08,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466468708, max timestamp=1732466468830Num uniques 1068; 2024-11-24T16:41:08,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-24T16:41:08,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:08,850 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:08,859 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#B#compaction#133 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:08,859 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/00046ac72a584a66bc100c05c3348e84 is 50, key is test_row_0/B:col10/1732466468601/Put/seqid=0 2024-11-24T16:41:08,887 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112498ce7007c8e74fbc8b5c8b0e1761b1fd_68db56cd5236f933146f262f370fc8cc store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:08,892 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112498ce7007c8e74fbc8b5c8b0e1761b1fd_68db56cd5236f933146f262f370fc8cc, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:08,893 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112498ce7007c8e74fbc8b5c8b0e1761b1fd_68db56cd5236f933146f262f370fc8cc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:08,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241632c654f6fc48de8b9f45de21427cfe_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466468831/Put/seqid=0 2024-11-24T16:41:08,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466468886, max timestamp=1732466468907Num uniques 859; 2024-11-24T16:41:08,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:41:08,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466468886, max timestamp=1732466468907Num uniques 859; 2024-11-24T16:41:08,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466468886, max timestamp=1732466468907Num uniques 859; 2024-11-24T16:41:08,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:41:08,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:41:08,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741976_1152 (size=12139) 2024-11-24T16:41:08,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741978_1154 (size=12304) 2024-11-24T16:41:08,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:08,941 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/00046ac72a584a66bc100c05c3348e84 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/00046ac72a584a66bc100c05c3348e84 2024-11-24T16:41:08,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241632c654f6fc48de8b9f45de21427cfe_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241632c654f6fc48de8b9f45de21427cfe_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:08,949 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4b68a5506fc549debb9724979446772f, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:08,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4b68a5506fc549debb9724979446772f is 175, key is test_row_0/A:col10/1732466468831/Put/seqid=0 2024-11-24T16:41:08,952 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/B of 68db56cd5236f933146f262f370fc8cc into 00046ac72a584a66bc100c05c3348e84(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:08,952 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:08,952 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/B, priority=12, startTime=1732466468820; duration=0sec 2024-11-24T16:41:08,952 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:08,952 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:B 2024-11-24T16:41:08,952 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741977_1153 (size=4469) 2024-11-24T16:41:08,954 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:08,954 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/C is initiating minor compaction (all files) 2024-11-24T16:41:08,954 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/C in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:08,955 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/194806cf526b448a9d9f7cb3a6e0cd2c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1b4e04a755d844878265fce3d5a3fda5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/05dcaeab001745649cec779a6a020725, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/84d61fb2a23e41bf80f3f35ffac55d05] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=46.9 K 2024-11-24T16:41:08,957 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 194806cf526b448a9d9f7cb3a6e0cd2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732466465723 2024-11-24T16:41:08,958 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b4e04a755d844878265fce3d5a3fda5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732466465865 2024-11-24T16:41:08,958 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#A#compaction#132 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:08,959 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/73a4fdf0d8f8491ea30d16d5e897f8cf is 175, key is test_row_0/A:col10/1732466468601/Put/seqid=0 2024-11-24T16:41:08,960 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 05dcaeab001745649cec779a6a020725, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732466466586 2024-11-24T16:41:08,961 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d61fb2a23e41bf80f3f35ffac55d05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732466468593 2024-11-24T16:41:08,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732466468929, max timestamp=1732466468960Num uniques 1118; 2024-11-24T16:41:08,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-24T16:41:08,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732466468929, max timestamp=1732466468960Num uniques 1118; 2024-11-24T16:41:08,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-24T16:41:08,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=368, cellSize=16848, totalHeapSize=38512, min timestamp=1732466468929, max timestamp=1732466468960Num uniques 1118; 2024-11-24T16:41:08,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=16848, suffix heap size=144240,new segment heap size=38512 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=368 2024-11-24T16:41:08,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741979_1155 (size=31105) 2024-11-24T16:41:08,973 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4b68a5506fc549debb9724979446772f 2024-11-24T16:41:08,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741980_1156 (size=31093) 2024-11-24T16:41:09,002 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/73a4fdf0d8f8491ea30d16d5e897f8cf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/73a4fdf0d8f8491ea30d16d5e897f8cf 2024-11-24T16:41:09,004 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:09,004 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/fbee557b0c624fd38e9d4e94bc6e21c9 is 50, key is test_row_0/C:col10/1732466468601/Put/seqid=0 2024-11-24T16:41:09,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/53cca20710924d81a46fca251cea450c is 50, key is test_row_0/B:col10/1732466468831/Put/seqid=0 2024-11-24T16:41:09,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466529001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,009 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/A of 68db56cd5236f933146f262f370fc8cc into 73a4fdf0d8f8491ea30d16d5e897f8cf(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:09,010 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:09,010 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/A, priority=12, startTime=1732466468820; duration=0sec 2024-11-24T16:41:09,012 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:09,014 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:A 2024-11-24T16:41:09,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466529004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466529007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466529007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466529008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741982_1158 (size=12139) 2024-11-24T16:41:09,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741981_1157 (size=12151) 2024-11-24T16:41:09,038 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/fbee557b0c624fd38e9d4e94bc6e21c9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fbee557b0c624fd38e9d4e94bc6e21c9 2024-11-24T16:41:09,045 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/C of 68db56cd5236f933146f262f370fc8cc into fbee557b0c624fd38e9d4e94bc6e21c9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:09,045 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:09,045 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/C, priority=12, startTime=1732466468825; duration=0sec 2024-11-24T16:41:09,045 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:09,045 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:C 2024-11-24T16:41:09,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466529114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466529123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466529124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466529124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466529124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466529324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466529334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466529334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466529334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466529334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/53cca20710924d81a46fca251cea450c 2024-11-24T16:41:09,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/9a0a5b782a304ac1ae0036e5d1f45bc3 is 50, key is test_row_0/C:col10/1732466468831/Put/seqid=0 2024-11-24T16:41:09,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741983_1159 (size=12151) 2024-11-24T16:41:09,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/9a0a5b782a304ac1ae0036e5d1f45bc3 2024-11-24T16:41:09,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4b68a5506fc549debb9724979446772f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4b68a5506fc549debb9724979446772f 2024-11-24T16:41:09,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4b68a5506fc549debb9724979446772f, entries=150, sequenceid=137, filesize=30.4 K 2024-11-24T16:41:09,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/53cca20710924d81a46fca251cea450c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/53cca20710924d81a46fca251cea450c 2024-11-24T16:41:09,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/53cca20710924d81a46fca251cea450c, entries=150, sequenceid=137, filesize=11.9 K 2024-11-24T16:41:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/9a0a5b782a304ac1ae0036e5d1f45bc3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/9a0a5b782a304ac1ae0036e5d1f45bc3 2024-11-24T16:41:09,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/9a0a5b782a304ac1ae0036e5d1f45bc3, entries=150, sequenceid=137, filesize=11.9 K 2024-11-24T16:41:09,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~100.50 KB/102912, currentSize=154.31 KB/158010 for 68db56cd5236f933146f262f370fc8cc in 661ms, sequenceid=137, compaction requested=false 2024-11-24T16:41:09,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:09,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=161.02 KB heapSize=445.08 KB 2024-11-24T16:41:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:09,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:09,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124abaf7eb0cdb84e029af9052554966d04_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466469640/Put/seqid=0 2024-11-24T16:41:09,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466529652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466529654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466529654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466529654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466529661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741984_1160 (size=14794) 2024-11-24T16:41:09,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466529764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466529774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T16:41:09,874 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:09,875 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-24T16:41:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T16:41:09,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:09,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:09,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:09,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466529973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T16:41:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466529984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:10,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:10,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:10,074 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124abaf7eb0cdb84e029af9052554966d04_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124abaf7eb0cdb84e029af9052554966d04_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:10,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/1a19acfa805f49fabc2106c9b1c9b29f, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:10,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/1a19acfa805f49fabc2106c9b1c9b29f is 175, key is test_row_0/A:col10/1732466469640/Put/seqid=0 2024-11-24T16:41:10,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741985_1161 (size=39749) 2024-11-24T16:41:10,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466530164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466530174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:10,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466530174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T16:41:10,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466530284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466530294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:10,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/1a19acfa805f49fabc2106c9b1c9b29f 2024-11-24T16:41:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T16:41:10,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/7076bdc31e6d443d981d9e5df42b13c8 is 50, key is test_row_0/B:col10/1732466469640/Put/seqid=0 2024-11-24T16:41:10,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741986_1162 (size=12151) 2024-11-24T16:41:10,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/7076bdc31e6d443d981d9e5df42b13c8 2024-11-24T16:41:10,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/ff63390deafd4de4b36298c3192a9460 is 50, key is test_row_0/C:col10/1732466469640/Put/seqid=0 2024-11-24T16:41:10,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741987_1163 (size=12151) 2024-11-24T16:41:10,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:10,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:10,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466530794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:10,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:10,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:10,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466530804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:10,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:10,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:11,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/ff63390deafd4de4b36298c3192a9460 2024-11-24T16:41:11,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/1a19acfa805f49fabc2106c9b1c9b29f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/1a19acfa805f49fabc2106c9b1c9b29f 2024-11-24T16:41:11,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/1a19acfa805f49fabc2106c9b1c9b29f, entries=200, sequenceid=194, filesize=38.8 K 2024-11-24T16:41:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T16:41:11,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/7076bdc31e6d443d981d9e5df42b13c8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7076bdc31e6d443d981d9e5df42b13c8 2024-11-24T16:41:11,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7076bdc31e6d443d981d9e5df42b13c8, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T16:41:11,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/ff63390deafd4de4b36298c3192a9460 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ff63390deafd4de4b36298c3192a9460 2024-11-24T16:41:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ff63390deafd4de4b36298c3192a9460, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T16:41:11,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.03 KB/455712, currentSize=26.84 KB/27480 for 68db56cd5236f933146f262f370fc8cc in 1399ms, sequenceid=194, compaction requested=true 2024-11-24T16:41:11,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:11,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:11,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:11,041 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:11,042 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,047 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:11,047 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:11,047 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/A is initiating minor compaction (all files) 2024-11-24T16:41:11,047 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/B is initiating minor compaction (all files) 2024-11-24T16:41:11,047 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/B in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:11,047 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/00046ac72a584a66bc100c05c3348e84, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/53cca20710924d81a46fca251cea450c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7076bdc31e6d443d981d9e5df42b13c8] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=35.6 K 2024-11-24T16:41:11,048 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/A in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:11,048 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/73a4fdf0d8f8491ea30d16d5e897f8cf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4b68a5506fc549debb9724979446772f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/1a19acfa805f49fabc2106c9b1c9b29f] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=99.6 K 2024-11-24T16:41:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,048 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,048 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/73a4fdf0d8f8491ea30d16d5e897f8cf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4b68a5506fc549debb9724979446772f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/1a19acfa805f49fabc2106c9b1c9b29f] 2024-11-24T16:41:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,050 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 00046ac72a584a66bc100c05c3348e84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732466468593 2024-11-24T16:41:11,050 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a4fdf0d8f8491ea30d16d5e897f8cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732466468593 2024-11-24T16:41:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,051 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b68a5506fc549debb9724979446772f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732466468709 2024-11-24T16:41:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,051 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 53cca20710924d81a46fca251cea450c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732466468709 2024-11-24T16:41:11,051 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a19acfa805f49fabc2106c9b1c9b29f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466468991 2024-11-24T16:41:11,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,052 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7076bdc31e6d443d981d9e5df42b13c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466468991 2024-11-24T16:41:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,083 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,084 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/aadb130699174f129fe4188cc1f8eea7 is 50, key is test_row_0/B:col10/1732466469640/Put/seqid=0 2024-11-24T16:41:11,085 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,089 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411245b32d7bec3104e55aa6b93569266aa2a_68db56cd5236f933146f262f370fc8cc store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741988_1164 (size=12391) 2024-11-24T16:41:11,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,094 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411245b32d7bec3104e55aa6b93569266aa2a_68db56cd5236f933146f262f370fc8cc, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:11,094 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245b32d7bec3104e55aa6b93569266aa2a_68db56cd5236f933146f262f370fc8cc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,100 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/aadb130699174f129fe4188cc1f8eea7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aadb130699174f129fe4188cc1f8eea7 2024-11-24T16:41:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,111 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/B of 68db56cd5236f933146f262f370fc8cc into aadb130699174f129fe4188cc1f8eea7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:11,111 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:11,111 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/B, priority=13, startTime=1732466471041; duration=0sec 2024-11-24T16:41:11,111 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:11,111 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:B 2024-11-24T16:41:11,111 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T16:41:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,113 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T16:41:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:11,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,116 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:11,116 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/C is initiating minor compaction (all files) 2024-11-24T16:41:11,116 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/C in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:11,116 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fbee557b0c624fd38e9d4e94bc6e21c9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/9a0a5b782a304ac1ae0036e5d1f45bc3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ff63390deafd4de4b36298c3192a9460] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=35.6 K 2024-11-24T16:41:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,117 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fbee557b0c624fd38e9d4e94bc6e21c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732466468593 2024-11-24T16:41:11,117 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0a5b782a304ac1ae0036e5d1f45bc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732466468709 2024-11-24T16:41:11,118 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ff63390deafd4de4b36298c3192a9460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466468991 2024-11-24T16:41:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741989_1165 (size=4469) 2024-11-24T16:41:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112477cfe2b07e75438295f9551414c25b3d_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466469646/Put/seqid=0 2024-11-24T16:41:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,150 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#C#compaction#144 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:11,151 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/fb5616d989424b44af8351ba5ddb034f is 50, key is test_row_0/C:col10/1732466469640/Put/seqid=0 2024-11-24T16:41:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741990_1166 (size=9814) 2024-11-24T16:41:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,171 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112477cfe2b07e75438295f9551414c25b3d_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477cfe2b07e75438295f9551414c25b3d_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:11,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3b132d8ed73f4ee4a99f6fd386e6e9c0, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3b132d8ed73f4ee4a99f6fd386e6e9c0 is 175, key is test_row_0/A:col10/1732466469646/Put/seqid=0 2024-11-24T16:41:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741991_1167 (size=12391) 2024-11-24T16:41:11,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:11,217 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/fb5616d989424b44af8351ba5ddb034f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fb5616d989424b44af8351ba5ddb034f 2024-11-24T16:41:11,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741992_1168 (size=22461) 2024-11-24T16:41:11,251 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/C of 68db56cd5236f933146f262f370fc8cc into fb5616d989424b44af8351ba5ddb034f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:11,251 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:11,251 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/C, priority=13, startTime=1732466471042; duration=0sec 2024-11-24T16:41:11,251 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:11,251 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:C 2024-11-24T16:41:11,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732466471262, max timestamp=1732466471285Num uniques 859; 2024-11-24T16:41:11,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732466471262, max timestamp=1732466471285Num uniques 859; 2024-11-24T16:41:11,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732466471262, max timestamp=1732466471285Num uniques 859; 2024-11-24T16:41:11,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T16:41:11,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T16:41:11,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T16:41:11,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466531320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466531322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466531323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466531434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466531434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466531434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,527 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:11,528 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/0a82235b7ee14fa2949cc140ea8df1eb is 175, key is test_row_0/A:col10/1732466469640/Put/seqid=0 2024-11-24T16:41:11,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741993_1169 (size=31345) 2024-11-24T16:41:11,540 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/0a82235b7ee14fa2949cc140ea8df1eb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0a82235b7ee14fa2949cc140ea8df1eb 2024-11-24T16:41:11,548 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/A of 68db56cd5236f933146f262f370fc8cc into 0a82235b7ee14fa2949cc140ea8df1eb(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:11,548 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:11,548 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/A, priority=13, startTime=1732466471041; duration=0sec 2024-11-24T16:41:11,548 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:11,549 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:A 2024-11-24T16:41:11,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3b132d8ed73f4ee4a99f6fd386e6e9c0 2024-11-24T16:41:11,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/695b79de95434fafb6d82e3c07ef25da is 50, key is test_row_0/B:col10/1732466469646/Put/seqid=0 2024-11-24T16:41:11,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741994_1170 (size=9757) 2024-11-24T16:41:11,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466531656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466531656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466531656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466531803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,811 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466531814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,819 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:11,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466531974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466531974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:11,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466531974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:11,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T16:41:12,054 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/695b79de95434fafb6d82e3c07ef25da 2024-11-24T16:41:12,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/e7e9591d0112430db343203ca2972389 is 50, key is test_row_0/C:col10/1732466469646/Put/seqid=0 2024-11-24T16:41:12,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741995_1171 (size=9757) 2024-11-24T16:41:12,075 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/e7e9591d0112430db343203ca2972389 2024-11-24T16:41:12,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3b132d8ed73f4ee4a99f6fd386e6e9c0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3b132d8ed73f4ee4a99f6fd386e6e9c0 2024-11-24T16:41:12,093 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3b132d8ed73f4ee4a99f6fd386e6e9c0, entries=100, sequenceid=202, filesize=21.9 K 2024-11-24T16:41:12,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/695b79de95434fafb6d82e3c07ef25da as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/695b79de95434fafb6d82e3c07ef25da 2024-11-24T16:41:12,100 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/695b79de95434fafb6d82e3c07ef25da, entries=100, sequenceid=202, filesize=9.5 K 2024-11-24T16:41:12,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/e7e9591d0112430db343203ca2972389 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e7e9591d0112430db343203ca2972389 2024-11-24T16:41:12,106 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e7e9591d0112430db343203ca2972389, entries=100, sequenceid=202, filesize=9.5 K 2024-11-24T16:41:12,108 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=161.02 KB/164880 for 68db56cd5236f933146f262f370fc8cc in 995ms, sequenceid=202, compaction requested=false 2024-11-24T16:41:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:12,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-24T16:41:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-24T16:41:12,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T16:41:12,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2330 sec 2024-11-24T16:41:12,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.2380 sec 2024-11-24T16:41:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:12,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=167.72 KB heapSize=466.55 KB 2024-11-24T16:41:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:41:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:41:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:12,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:41:12,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112461207a03f42148b393b1b3ce55a06ce1_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466472486/Put/seqid=0 2024-11-24T16:41:12,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466532501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466532505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466532510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741996_1172 (size=14794) 2024-11-24T16:41:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466532614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466532614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466532624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466532824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466532824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466532834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:12,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:12,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112461207a03f42148b393b1b3ce55a06ce1_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112461207a03f42148b393b1b3ce55a06ce1_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:12,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7ff823fead2140348459905df882779a, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:12,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7ff823fead2140348459905df882779a is 175, key is test_row_0/A:col10/1732466472486/Put/seqid=0 2024-11-24T16:41:12,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741997_1173 (size=39749) 2024-11-24T16:41:12,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7ff823fead2140348459905df882779a 2024-11-24T16:41:12,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/07b16ee8322a40119eb99c0fe1affe29 is 50, key is test_row_0/B:col10/1732466472486/Put/seqid=0 2024-11-24T16:41:12,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741998_1174 (size=12151) 2024-11-24T16:41:12,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/07b16ee8322a40119eb99c0fe1affe29 2024-11-24T16:41:13,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0bae676eadcf443abadb76360097b957 is 50, key is test_row_0/C:col10/1732466472486/Put/seqid=0 2024-11-24T16:41:13,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741999_1175 (size=12151) 2024-11-24T16:41:13,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466533134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466533134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466533143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0bae676eadcf443abadb76360097b957 2024-11-24T16:41:13,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7ff823fead2140348459905df882779a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7ff823fead2140348459905df882779a 2024-11-24T16:41:13,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7ff823fead2140348459905df882779a, entries=200, sequenceid=242, filesize=38.8 K 2024-11-24T16:41:13,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/07b16ee8322a40119eb99c0fe1affe29 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/07b16ee8322a40119eb99c0fe1affe29 2024-11-24T16:41:13,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/07b16ee8322a40119eb99c0fe1affe29, entries=150, sequenceid=242, filesize=11.9 K 2024-11-24T16:41:13,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0bae676eadcf443abadb76360097b957 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0bae676eadcf443abadb76360097b957 2024-11-24T16:41:13,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0bae676eadcf443abadb76360097b957, entries=150, sequenceid=242, filesize=11.9 K 2024-11-24T16:41:13,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~466.50 KB/477696, currentSize=26.84 KB/27480 for 68db56cd5236f933146f262f370fc8cc in 975ms, sequenceid=242, compaction requested=true 2024-11-24T16:41:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:13,464 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:13,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T16:41:13,464 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:13,465 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:13,465 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/A is initiating minor compaction (all files) 2024-11-24T16:41:13,465 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/A in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:13,466 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0a82235b7ee14fa2949cc140ea8df1eb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3b132d8ed73f4ee4a99f6fd386e6e9c0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7ff823fead2140348459905df882779a] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=91.4 K 2024-11-24T16:41:13,466 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:13,466 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:13,466 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/B is initiating minor compaction (all files) 2024-11-24T16:41:13,466 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0a82235b7ee14fa2949cc140ea8df1eb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3b132d8ed73f4ee4a99f6fd386e6e9c0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7ff823fead2140348459905df882779a] 2024-11-24T16:41:13,466 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/B in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:13,466 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aadb130699174f129fe4188cc1f8eea7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/695b79de95434fafb6d82e3c07ef25da, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/07b16ee8322a40119eb99c0fe1affe29] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=33.5 K 2024-11-24T16:41:13,468 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a82235b7ee14fa2949cc140ea8df1eb, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466468991 2024-11-24T16:41:13,469 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting aadb130699174f129fe4188cc1f8eea7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466468991 2024-11-24T16:41:13,472 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b132d8ed73f4ee4a99f6fd386e6e9c0, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732466469646 2024-11-24T16:41:13,473 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ff823fead2140348459905df882779a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732466471317 2024-11-24T16:41:13,473 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 695b79de95434fafb6d82e3c07ef25da, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732466469646 2024-11-24T16:41:13,474 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 07b16ee8322a40119eb99c0fe1affe29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732466471317 2024-11-24T16:41:13,489 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:13,507 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:13,507 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/aa9f9d4c256b462cb53a7848289095ab is 50, key is test_row_0/B:col10/1732466472486/Put/seqid=0 2024-11-24T16:41:13,515 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124a37360c08167483a9583e6dd77428912_68db56cd5236f933146f262f370fc8cc store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:13,517 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124a37360c08167483a9583e6dd77428912_68db56cd5236f933146f262f370fc8cc, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:13,518 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a37360c08167483a9583e6dd77428912_68db56cd5236f933146f262f370fc8cc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:13,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742000_1176 (size=12493) 2024-11-24T16:41:13,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742001_1177 (size=4469) 2024-11-24T16:41:13,552 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/aa9f9d4c256b462cb53a7848289095ab as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aa9f9d4c256b462cb53a7848289095ab 2024-11-24T16:41:13,561 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/B of 68db56cd5236f933146f262f370fc8cc into aa9f9d4c256b462cb53a7848289095ab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:13,561 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:13,561 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/B, priority=13, startTime=1732466473464; duration=0sec 2024-11-24T16:41:13,562 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:13,562 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:B 2024-11-24T16:41:13,562 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:13,572 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:13,572 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/C is initiating minor compaction (all files) 2024-11-24T16:41:13,572 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/C in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:13,578 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fb5616d989424b44af8351ba5ddb034f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e7e9591d0112430db343203ca2972389, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0bae676eadcf443abadb76360097b957] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=33.5 K 2024-11-24T16:41:13,585 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fb5616d989424b44af8351ba5ddb034f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466468991 2024-11-24T16:41:13,586 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e9591d0112430db343203ca2972389, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732466469646 2024-11-24T16:41:13,586 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bae676eadcf443abadb76360097b957, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732466471317 2024-11-24T16:41:13,612 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#C#compaction#152 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:13,613 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/f8917d18da6e46138d6e9d0f1cbb510c is 50, key is test_row_0/C:col10/1732466472486/Put/seqid=0 2024-11-24T16:41:13,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742002_1178 (size=12493) 2024-11-24T16:41:13,642 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/f8917d18da6e46138d6e9d0f1cbb510c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f8917d18da6e46138d6e9d0f1cbb510c 2024-11-24T16:41:13,650 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/C of 68db56cd5236f933146f262f370fc8cc into f8917d18da6e46138d6e9d0f1cbb510c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:13,651 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:13,651 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/C, priority=13, startTime=1732466473464; duration=0sec 2024-11-24T16:41:13,651 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:13,651 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:C 2024-11-24T16:41:13,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:41:13,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:13,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:13,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:13,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:13,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:13,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:13,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b270b6cd1e924cc5b63616ef4b1ca613_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466473658/Put/seqid=0 2024-11-24T16:41:13,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742003_1179 (size=12304) 2024-11-24T16:41:13,728 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:13,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b270b6cd1e924cc5b63616ef4b1ca613_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b270b6cd1e924cc5b63616ef4b1ca613_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:13,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3388c34bb8cb426594187683f8136304, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:13,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3388c34bb8cb426594187683f8136304 is 175, key is test_row_0/A:col10/1732466473658/Put/seqid=0 2024-11-24T16:41:13,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466473709, max timestamp=1732466473734Num uniques 859; 2024-11-24T16:41:13,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:13,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466473709, max timestamp=1732466473733Num uniques 859; 2024-11-24T16:41:13,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:13,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466473709, max timestamp=1732466473733Num uniques 859; 2024-11-24T16:41:13,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:13,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742004_1180 (size=31105) 2024-11-24T16:41:13,750 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3388c34bb8cb426594187683f8136304 2024-11-24T16:41:13,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/f7265bb169f6412a893a9f044355ec76 is 50, key is test_row_0/B:col10/1732466473658/Put/seqid=0 2024-11-24T16:41:13,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742005_1181 (size=12151) 2024-11-24T16:41:13,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466533778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466533780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466473753, max timestamp=1732466473779Num uniques 1018; 2024-11-24T16:41:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T16:41:13,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466473753, max timestamp=1732466473779Num uniques 1018; 2024-11-24T16:41:13,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T16:41:13,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466473753, max timestamp=1732466473779Num uniques 1018; 2024-11-24T16:41:13,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T16:41:13,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466533844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466533845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,854 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466533849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466533888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466533893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,948 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#A#compaction#150 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:13,949 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/17cb0dd1dd3a4edfabc5c182db530143 is 175, key is test_row_0/A:col10/1732466472486/Put/seqid=0 2024-11-24T16:41:13,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466533965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466533965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:13,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:13,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742006_1182 (size=31447) 2024-11-24T16:41:13,986 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/17cb0dd1dd3a4edfabc5c182db530143 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/17cb0dd1dd3a4edfabc5c182db530143 2024-11-24T16:41:13,992 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/A of 68db56cd5236f933146f262f370fc8cc into 17cb0dd1dd3a4edfabc5c182db530143(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:13,992 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:13,992 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/A, priority=13, startTime=1732466473464; duration=0sec 2024-11-24T16:41:13,992 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:13,992 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:A 2024-11-24T16:41:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T16:41:14,044 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:14,045 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-24T16:41:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T16:41:14,047 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:14,048 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:14,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:14,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466534104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466534104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T16:41:14,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/f7265bb169f6412a893a9f044355ec76 2024-11-24T16:41:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466534174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466534174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T16:41:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:14,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/ddde9ad3fd31425ebbba41c5f6f7e24d is 50, key is test_row_0/C:col10/1732466473658/Put/seqid=0 2024-11-24T16:41:14,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742007_1183 (size=12151) 2024-11-24T16:41:14,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T16:41:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:14,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T16:41:14,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466534424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466534424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466534483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466534493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T16:41:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:14,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:14,614 INFO [master/e335232f7491:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T16:41:14,614 INFO [master/e335232f7491:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T16:41:14,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/ddde9ad3fd31425ebbba41c5f6f7e24d 2024-11-24T16:41:14,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/3388c34bb8cb426594187683f8136304 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3388c34bb8cb426594187683f8136304 2024-11-24T16:41:14,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3388c34bb8cb426594187683f8136304, entries=150, sequenceid=254, filesize=30.4 K 2024-11-24T16:41:14,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/f7265bb169f6412a893a9f044355ec76 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f7265bb169f6412a893a9f044355ec76 2024-11-24T16:41:14,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f7265bb169f6412a893a9f044355ec76, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T16:41:14,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/ddde9ad3fd31425ebbba41c5f6f7e24d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ddde9ad3fd31425ebbba41c5f6f7e24d 2024-11-24T16:41:14,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ddde9ad3fd31425ebbba41c5f6f7e24d, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T16:41:14,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 68db56cd5236f933146f262f370fc8cc in 1003ms, sequenceid=254, compaction requested=false 2024-11-24T16:41:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:14,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T16:41:14,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:14,665 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-24T16:41:14,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:14,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:14,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:14,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T16:41:14,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c5bbbcbc55be4ee18b1d47a67d223d85_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466473847/Put/seqid=0 2024-11-24T16:41:14,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742008_1184 (size=12454) 2024-11-24T16:41:14,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:14,752 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c5bbbcbc55be4ee18b1d47a67d223d85_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5bbbcbc55be4ee18b1d47a67d223d85_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:14,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7edc932681f546d8b476a3801e89c0c7, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7edc932681f546d8b476a3801e89c0c7 is 175, key is test_row_0/A:col10/1732466473847/Put/seqid=0 2024-11-24T16:41:14,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742009_1185 (size=31255) 2024-11-24T16:41:14,760 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7edc932681f546d8b476a3801e89c0c7 2024-11-24T16:41:14,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/eb171312039747caab8bab6ea3349013 is 50, key is test_row_0/B:col10/1732466473847/Put/seqid=0 2024-11-24T16:41:14,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742010_1186 (size=12301) 2024-11-24T16:41:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:14,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466534947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:14,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466534956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466534993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466535003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466535064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466535073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T16:41:15,187 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/eb171312039747caab8bab6ea3349013 2024-11-24T16:41:15,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/22a498b65f504e81aa44f82628460b18 is 50, key is test_row_0/C:col10/1732466473847/Put/seqid=0 2024-11-24T16:41:15,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742011_1187 (size=12301) 2024-11-24T16:41:15,237 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/22a498b65f504e81aa44f82628460b18 2024-11-24T16:41:15,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/7edc932681f546d8b476a3801e89c0c7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7edc932681f546d8b476a3801e89c0c7 2024-11-24T16:41:15,255 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7edc932681f546d8b476a3801e89c0c7, entries=150, sequenceid=308, filesize=30.5 K 2024-11-24T16:41:15,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/eb171312039747caab8bab6ea3349013 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/eb171312039747caab8bab6ea3349013 2024-11-24T16:41:15,263 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/eb171312039747caab8bab6ea3349013, entries=150, sequenceid=308, filesize=12.0 K 2024-11-24T16:41:15,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/22a498b65f504e81aa44f82628460b18 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/22a498b65f504e81aa44f82628460b18 2024-11-24T16:41:15,272 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/22a498b65f504e81aa44f82628460b18, entries=150, sequenceid=308, filesize=12.0 K 2024-11-24T16:41:15,274 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=46.96 KB/48090 for 68db56cd5236f933146f262f370fc8cc in 608ms, sequenceid=308, compaction requested=true 2024-11-24T16:41:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-24T16:41:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-24T16:41:15,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-24T16:41:15,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2290 sec 2024-11-24T16:41:15,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2360 sec 2024-11-24T16:41:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:15,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:41:15,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:15,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:15,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:15,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:15,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:15,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:15,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247364f7f242ed4d1f999647f2d3d295c1_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466475287/Put/seqid=0 2024-11-24T16:41:15,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742012_1188 (size=14994) 2024-11-24T16:41:15,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466535348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466535351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466535463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466535463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466535674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466535674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:15,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247364f7f242ed4d1f999647f2d3d295c1_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247364f7f242ed4d1f999647f2d3d295c1_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:15,733 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/f92fc4ca826d40c98c4f2347265b941d, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:15,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/f92fc4ca826d40c98c4f2347265b941d is 175, key is test_row_0/A:col10/1732466475287/Put/seqid=0 2024-11-24T16:41:15,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742013_1189 (size=39949) 2024-11-24T16:41:15,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466535984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466535984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466536013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466536014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,018 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:16,018 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,145 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/f92fc4ca826d40c98c4f2347265b941d 2024-11-24T16:41:16,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/19d605272154439ebdf366891d82542e is 50, key is test_row_0/B:col10/1732466475287/Put/seqid=0 2024-11-24T16:41:16,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742014_1190 (size=12301) 2024-11-24T16:41:16,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/19d605272154439ebdf366891d82542e 2024-11-24T16:41:16,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0c6c62c01dd641a3b9d72ee16ea66cb0 is 50, key is test_row_0/C:col10/1732466475287/Put/seqid=0 2024-11-24T16:41:16,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742015_1191 (size=12301) 2024-11-24T16:41:16,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0c6c62c01dd641a3b9d72ee16ea66cb0 2024-11-24T16:41:16,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/f92fc4ca826d40c98c4f2347265b941d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/f92fc4ca826d40c98c4f2347265b941d 2024-11-24T16:41:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T16:41:16,205 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:16,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-24T16:41:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T16:41:16,211 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:16,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/f92fc4ca826d40c98c4f2347265b941d, entries=200, sequenceid=320, filesize=39.0 K 2024-11-24T16:41:16,212 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:16,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:16,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/19d605272154439ebdf366891d82542e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/19d605272154439ebdf366891d82542e 2024-11-24T16:41:16,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/19d605272154439ebdf366891d82542e, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T16:41:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0c6c62c01dd641a3b9d72ee16ea66cb0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c6c62c01dd641a3b9d72ee16ea66cb0 2024-11-24T16:41:16,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c6c62c01dd641a3b9d72ee16ea66cb0, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T16:41:16,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 68db56cd5236f933146f262f370fc8cc in 938ms, sequenceid=320, compaction requested=true 2024-11-24T16:41:16,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:16,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:16,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:16,228 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:16,229 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:16,230 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49246 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:16,230 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/B is initiating minor compaction (all files) 2024-11-24T16:41:16,230 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/B in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:16,231 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aa9f9d4c256b462cb53a7848289095ab, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f7265bb169f6412a893a9f044355ec76, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/eb171312039747caab8bab6ea3349013, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/19d605272154439ebdf366891d82542e] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=48.1 K 2024-11-24T16:41:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:16,233 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9f9d4c256b462cb53a7848289095ab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732466471317 2024-11-24T16:41:16,233 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133756 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:16,233 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f7265bb169f6412a893a9f044355ec76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732466472495 2024-11-24T16:41:16,233 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/A is initiating minor compaction (all files) 2024-11-24T16:41:16,233 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/A in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:16,233 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/17cb0dd1dd3a4edfabc5c182db530143, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3388c34bb8cb426594187683f8136304, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7edc932681f546d8b476a3801e89c0c7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/f92fc4ca826d40c98c4f2347265b941d] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=130.6 K 2024-11-24T16:41:16,233 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:16,233 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/17cb0dd1dd3a4edfabc5c182db530143, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3388c34bb8cb426594187683f8136304, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7edc932681f546d8b476a3801e89c0c7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/f92fc4ca826d40c98c4f2347265b941d] 2024-11-24T16:41:16,234 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting eb171312039747caab8bab6ea3349013, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732466473815 2024-11-24T16:41:16,236 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17cb0dd1dd3a4edfabc5c182db530143, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732466471317 2024-11-24T16:41:16,237 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d605272154439ebdf366891d82542e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732466474945 2024-11-24T16:41:16,237 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3388c34bb8cb426594187683f8136304, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732466472495 2024-11-24T16:41:16,238 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7edc932681f546d8b476a3801e89c0c7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732466473815 2024-11-24T16:41:16,238 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting f92fc4ca826d40c98c4f2347265b941d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732466474936 2024-11-24T16:41:16,258 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:16,262 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#B#compaction#162 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:16,262 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/f046654da2654f0fbbadf635708fe51e is 50, key is test_row_0/B:col10/1732466475287/Put/seqid=0 2024-11-24T16:41:16,266 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112476d82950a29d43389ffa8f52ddf31662_68db56cd5236f933146f262f370fc8cc store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:16,269 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112476d82950a29d43389ffa8f52ddf31662_68db56cd5236f933146f262f370fc8cc, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:16,269 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112476d82950a29d43389ffa8f52ddf31662_68db56cd5236f933146f262f370fc8cc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:16,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742016_1192 (size=12779) 2024-11-24T16:41:16,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742017_1193 (size=4469) 2024-11-24T16:41:16,299 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#A#compaction#163 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:16,300 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/152eae1619d64c618f3bc3add22f544e is 175, key is test_row_0/A:col10/1732466475287/Put/seqid=0 2024-11-24T16:41:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T16:41:16,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742018_1194 (size=31733) 2024-11-24T16:41:16,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T16:41:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:16,365 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:41:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:16,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243e30e2db7e494f199b8fdccb188f9f73_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466475340/Put/seqid=0 2024-11-24T16:41:16,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742019_1195 (size=12454) 2024-11-24T16:41:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T16:41:16,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466536525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466536526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466536634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466536634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,692 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/f046654da2654f0fbbadf635708fe51e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f046654da2654f0fbbadf635708fe51e 2024-11-24T16:41:16,704 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/B of 68db56cd5236f933146f262f370fc8cc into f046654da2654f0fbbadf635708fe51e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:16,704 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:16,704 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/B, priority=12, startTime=1732466476228; duration=0sec 2024-11-24T16:41:16,704 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:16,704 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:B 2024-11-24T16:41:16,704 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:16,706 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49246 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:16,706 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/C is initiating minor compaction (all files) 2024-11-24T16:41:16,706 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/C in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:16,706 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f8917d18da6e46138d6e9d0f1cbb510c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ddde9ad3fd31425ebbba41c5f6f7e24d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/22a498b65f504e81aa44f82628460b18, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c6c62c01dd641a3b9d72ee16ea66cb0] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=48.1 K 2024-11-24T16:41:16,707 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f8917d18da6e46138d6e9d0f1cbb510c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732466471317 2024-11-24T16:41:16,707 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ddde9ad3fd31425ebbba41c5f6f7e24d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732466472495 2024-11-24T16:41:16,708 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a498b65f504e81aa44f82628460b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732466473815 2024-11-24T16:41:16,708 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c6c62c01dd641a3b9d72ee16ea66cb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732466474945 2024-11-24T16:41:16,731 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/152eae1619d64c618f3bc3add22f544e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/152eae1619d64c618f3bc3add22f544e 2024-11-24T16:41:16,741 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#C#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:16,742 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/cd4ab22f79c94742ab749f21fb8448e3 is 50, key is test_row_0/C:col10/1732466475287/Put/seqid=0 2024-11-24T16:41:16,743 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/A of 68db56cd5236f933146f262f370fc8cc into 152eae1619d64c618f3bc3add22f544e(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:16,743 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:16,743 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/A, priority=12, startTime=1732466476228; duration=0sec 2024-11-24T16:41:16,743 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:16,743 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:A 2024-11-24T16:41:16,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742020_1196 (size=12779) 2024-11-24T16:41:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:16,807 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243e30e2db7e494f199b8fdccb188f9f73_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243e30e2db7e494f199b8fdccb188f9f73_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:16,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/d191a838ba644805b7cf1e1c1a86096e, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:16,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/d191a838ba644805b7cf1e1c1a86096e is 175, key is test_row_0/A:col10/1732466475340/Put/seqid=0 2024-11-24T16:41:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T16:41:16,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742021_1197 (size=31255) 2024-11-24T16:41:16,850 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=340, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/d191a838ba644805b7cf1e1c1a86096e 2024-11-24T16:41:16,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466536844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466536853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:16,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:16,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/3ac76cd353db42738815a7d6d0c64365 is 50, key is test_row_0/B:col10/1732466475340/Put/seqid=0 2024-11-24T16:41:16,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742022_1198 (size=12301) 2024-11-24T16:41:16,896 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/3ac76cd353db42738815a7d6d0c64365 2024-11-24T16:41:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/f027f026970a4015803d7490822d0773 is 50, key is test_row_0/C:col10/1732466475340/Put/seqid=0 2024-11-24T16:41:16,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742023_1199 (size=12301) 2024-11-24T16:41:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466537154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466537163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:17,184 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/cd4ab22f79c94742ab749f21fb8448e3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/cd4ab22f79c94742ab749f21fb8448e3 2024-11-24T16:41:17,192 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/C of 68db56cd5236f933146f262f370fc8cc into cd4ab22f79c94742ab749f21fb8448e3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:17,192 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:17,192 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/C, priority=12, startTime=1732466476232; duration=0sec 2024-11-24T16:41:17,192 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:17,192 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:C 2024-11-24T16:41:17,326 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/f027f026970a4015803d7490822d0773 2024-11-24T16:41:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T16:41:17,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/d191a838ba644805b7cf1e1c1a86096e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/d191a838ba644805b7cf1e1c1a86096e 2024-11-24T16:41:17,351 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/d191a838ba644805b7cf1e1c1a86096e, entries=150, sequenceid=340, filesize=30.5 K 2024-11-24T16:41:17,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/3ac76cd353db42738815a7d6d0c64365 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3ac76cd353db42738815a7d6d0c64365 2024-11-24T16:41:17,358 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3ac76cd353db42738815a7d6d0c64365, entries=150, sequenceid=340, filesize=12.0 K 2024-11-24T16:41:17,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/f027f026970a4015803d7490822d0773 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f027f026970a4015803d7490822d0773 2024-11-24T16:41:17,368 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f027f026970a4015803d7490822d0773, entries=150, sequenceid=340, filesize=12.0 K 2024-11-24T16:41:17,370 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 68db56cd5236f933146f262f370fc8cc in 1005ms, sequenceid=340, compaction requested=false 2024-11-24T16:41:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-24T16:41:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-24T16:41:17,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-24T16:41:17,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1650 sec 2024-11-24T16:41:17,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1850 sec 2024-11-24T16:41:17,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:41:17,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:17,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:17,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:17,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:17,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:17,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:17,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241dafb4eafaa34e52ab45c5e15835d1ff_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466476524/Put/seqid=0 2024-11-24T16:41:17,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466537703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:17,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742024_1200 (size=14994) 2024-11-24T16:41:17,710 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466537707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:17,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241dafb4eafaa34e52ab45c5e15835d1ff_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241dafb4eafaa34e52ab45c5e15835d1ff_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:17,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/8c0b1f62a5a348269bf4fc4801d64a01, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:17,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/8c0b1f62a5a348269bf4fc4801d64a01 is 175, key is test_row_0/A:col10/1732466476524/Put/seqid=0 2024-11-24T16:41:17,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742025_1201 (size=39949) 2024-11-24T16:41:17,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466537813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:17,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466537824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58248 deadline: 1732466537883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:17,887 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466538023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466538023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,028 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466538034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466538034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,037 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/8c0b1f62a5a348269bf4fc4801d64a01 2024-11-24T16:41:18,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/9586da7a1dc94e6ca63dc24de4a021bb is 50, key is test_row_0/B:col10/1732466476524/Put/seqid=0 2024-11-24T16:41:18,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742026_1202 (size=12301) 2024-11-24T16:41:18,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466538334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466538343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T16:41:18,354 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:18,354 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-24T16:41:18,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T16:41:18,356 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:18,357 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:18,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T16:41:18,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T16:41:18,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:18,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:18,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:18,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:18,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:18,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/9586da7a1dc94e6ca63dc24de4a021bb 2024-11-24T16:41:18,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0513afdeae0949f9974291a6bd5bdc85 is 50, key is test_row_0/C:col10/1732466476524/Put/seqid=0 2024-11-24T16:41:18,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742027_1203 (size=12301) 2024-11-24T16:41:18,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0513afdeae0949f9974291a6bd5bdc85 2024-11-24T16:41:18,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/8c0b1f62a5a348269bf4fc4801d64a01 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/8c0b1f62a5a348269bf4fc4801d64a01 2024-11-24T16:41:18,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/8c0b1f62a5a348269bf4fc4801d64a01, entries=200, sequenceid=356, filesize=39.0 K 2024-11-24T16:41:18,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/9586da7a1dc94e6ca63dc24de4a021bb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/9586da7a1dc94e6ca63dc24de4a021bb 2024-11-24T16:41:18,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/9586da7a1dc94e6ca63dc24de4a021bb, entries=150, sequenceid=356, filesize=12.0 K 2024-11-24T16:41:18,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0513afdeae0949f9974291a6bd5bdc85 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0513afdeae0949f9974291a6bd5bdc85 2024-11-24T16:41:18,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0513afdeae0949f9974291a6bd5bdc85, entries=150, sequenceid=356, filesize=12.0 K 2024-11-24T16:41:18,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 68db56cd5236f933146f262f370fc8cc in 920ms, sequenceid=356, compaction requested=true 2024-11-24T16:41:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:18,586 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:18,586 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:18,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:18,587 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:18,587 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:18,587 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/A is initiating minor compaction (all files) 2024-11-24T16:41:18,587 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/B is initiating minor compaction (all files) 2024-11-24T16:41:18,588 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/B in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:18,588 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/A in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:18,588 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f046654da2654f0fbbadf635708fe51e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3ac76cd353db42738815a7d6d0c64365, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/9586da7a1dc94e6ca63dc24de4a021bb] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=36.5 K 2024-11-24T16:41:18,588 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/152eae1619d64c618f3bc3add22f544e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/d191a838ba644805b7cf1e1c1a86096e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/8c0b1f62a5a348269bf4fc4801d64a01] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=100.5 K 2024-11-24T16:41:18,588 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:18,588 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/152eae1619d64c618f3bc3add22f544e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/d191a838ba644805b7cf1e1c1a86096e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/8c0b1f62a5a348269bf4fc4801d64a01] 2024-11-24T16:41:18,588 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152eae1619d64c618f3bc3add22f544e, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732466474945 2024-11-24T16:41:18,588 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f046654da2654f0fbbadf635708fe51e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732466474945 2024-11-24T16:41:18,589 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting d191a838ba644805b7cf1e1c1a86096e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732466475339 2024-11-24T16:41:18,589 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac76cd353db42738815a7d6d0c64365, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732466475339 2024-11-24T16:41:18,589 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c0b1f62a5a348269bf4fc4801d64a01, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732466476509 2024-11-24T16:41:18,589 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 9586da7a1dc94e6ca63dc24de4a021bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732466476509 2024-11-24T16:41:18,597 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:18,599 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#B#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:18,600 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/3735d08d1b3e4384a30f127ff5d9f952 is 50, key is test_row_0/B:col10/1732466476524/Put/seqid=0 2024-11-24T16:41:18,601 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124e157cb1d236d4eacba935dd606d5b05a_68db56cd5236f933146f262f370fc8cc store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:18,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742028_1204 (size=12881) 2024-11-24T16:41:18,603 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124e157cb1d236d4eacba935dd606d5b05a_68db56cd5236f933146f262f370fc8cc, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:18,604 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e157cb1d236d4eacba935dd606d5b05a_68db56cd5236f933146f262f370fc8cc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:18,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742029_1205 (size=4469) 2024-11-24T16:41:18,610 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/3735d08d1b3e4384a30f127ff5d9f952 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3735d08d1b3e4384a30f127ff5d9f952 2024-11-24T16:41:18,615 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/B of 68db56cd5236f933146f262f370fc8cc into 3735d08d1b3e4384a30f127ff5d9f952(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:18,615 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:18,615 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/B, priority=13, startTime=1732466478586; duration=0sec 2024-11-24T16:41:18,615 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:18,615 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:B 2024-11-24T16:41:18,615 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:18,617 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:18,617 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/C is initiating minor compaction (all files) 2024-11-24T16:41:18,617 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/C in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:18,617 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/cd4ab22f79c94742ab749f21fb8448e3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f027f026970a4015803d7490822d0773, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0513afdeae0949f9974291a6bd5bdc85] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=36.5 K 2024-11-24T16:41:18,617 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4ab22f79c94742ab749f21fb8448e3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732466474945 2024-11-24T16:41:18,618 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f027f026970a4015803d7490822d0773, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732466475339 2024-11-24T16:41:18,618 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0513afdeae0949f9974291a6bd5bdc85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732466476509 2024-11-24T16:41:18,628 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:18,628 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0c2d7ceb8b1b480ca327135dadf26edf is 50, key is test_row_0/C:col10/1732466476524/Put/seqid=0 2024-11-24T16:41:18,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742030_1206 (size=12881) 2024-11-24T16:41:18,637 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0c2d7ceb8b1b480ca327135dadf26edf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c2d7ceb8b1b480ca327135dadf26edf 2024-11-24T16:41:18,643 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/C of 68db56cd5236f933146f262f370fc8cc into 0c2d7ceb8b1b480ca327135dadf26edf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:18,644 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:18,644 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/C, priority=13, startTime=1732466478586; duration=0sec 2024-11-24T16:41:18,644 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:18,644 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:C 2024-11-24T16:41:18,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T16:41:18,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:18,663 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T16:41:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:18,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:18,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c3e53c14bfe446fd9e3dd29ea8bac85e_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466477688/Put/seqid=0 2024-11-24T16:41:18,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742031_1207 (size=12454) 2024-11-24T16:41:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T16:41:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:18,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:18,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466538867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466538875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T16:41:18,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466538984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466538984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:19,010 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#A#compaction#171 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:19,010 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/71c3bf50ccf341a2be1ceb24953c3f27 is 175, key is test_row_0/A:col10/1732466476524/Put/seqid=0 2024-11-24T16:41:19,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742032_1208 (size=31835) 2024-11-24T16:41:19,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:19,079 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c3e53c14bfe446fd9e3dd29ea8bac85e_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c3e53c14bfe446fd9e3dd29ea8bac85e_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:19,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/66a4a112ae29434098e4dda70300047d, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:19,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/66a4a112ae29434098e4dda70300047d is 175, key is test_row_0/A:col10/1732466477688/Put/seqid=0 2024-11-24T16:41:19,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742033_1209 (size=31255) 2024-11-24T16:41:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466539193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466539193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:19,421 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/71c3bf50ccf341a2be1ceb24953c3f27 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/71c3bf50ccf341a2be1ceb24953c3f27 2024-11-24T16:41:19,426 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/A of 68db56cd5236f933146f262f370fc8cc into 71c3bf50ccf341a2be1ceb24953c3f27(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:19,426 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:19,426 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/A, priority=13, startTime=1732466478586; duration=0sec 2024-11-24T16:41:19,427 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:19,427 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:A 2024-11-24T16:41:19,486 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/66a4a112ae29434098e4dda70300047d 2024-11-24T16:41:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T16:41:19,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/a0f536c904594f2f83dc00429856b811 is 50, key is test_row_0/B:col10/1732466477688/Put/seqid=0 2024-11-24T16:41:19,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742034_1210 (size=12301) 2024-11-24T16:41:19,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466539504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:19,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466539513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:19,898 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/a0f536c904594f2f83dc00429856b811 2024-11-24T16:41:19,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/e2706aba6ead4915a18f93d40b366d75 is 50, key is test_row_0/C:col10/1732466477688/Put/seqid=0 2024-11-24T16:41:19,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742035_1211 (size=12301) 2024-11-24T16:41:20,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466540014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:20,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:20,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466540024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:20,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:20,318 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/e2706aba6ead4915a18f93d40b366d75 2024-11-24T16:41:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/66a4a112ae29434098e4dda70300047d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/66a4a112ae29434098e4dda70300047d 2024-11-24T16:41:20,329 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/66a4a112ae29434098e4dda70300047d, entries=150, sequenceid=377, filesize=30.5 K 2024-11-24T16:41:20,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/a0f536c904594f2f83dc00429856b811 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a0f536c904594f2f83dc00429856b811 2024-11-24T16:41:20,334 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a0f536c904594f2f83dc00429856b811, entries=150, sequenceid=377, filesize=12.0 K 2024-11-24T16:41:20,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/e2706aba6ead4915a18f93d40b366d75 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e2706aba6ead4915a18f93d40b366d75 2024-11-24T16:41:20,341 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e2706aba6ead4915a18f93d40b366d75, entries=150, sequenceid=377, filesize=12.0 K 2024-11-24T16:41:20,342 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 68db56cd5236f933146f262f370fc8cc in 1680ms, sequenceid=377, compaction requested=false 2024-11-24T16:41:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T16:41:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T16:41:20,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T16:41:20,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9870 sec 2024-11-24T16:41:20,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.9920 sec 2024-11-24T16:41:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T16:41:20,504 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:20,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-24T16:41:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T16:41:20,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:20,507 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:20,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T16:41:20,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T16:41:20,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:20,659 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:41:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:20,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241dc37f9cb1814128ad00de278afd0ee4_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466478874/Put/seqid=0 2024-11-24T16:41:20,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742036_1212 (size=12454) 2024-11-24T16:41:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T16:41:21,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:21,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:21,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466541064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:21,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466541071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,080 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241dc37f9cb1814128ad00de278afd0ee4_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241dc37f9cb1814128ad00de278afd0ee4_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:21,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/90168a17246f43df890b8e9918401aa7, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/90168a17246f43df890b8e9918401aa7 is 175, key is test_row_0/A:col10/1732466478874/Put/seqid=0 2024-11-24T16:41:21,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742037_1213 (size=31255) 2024-11-24T16:41:21,086 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/90168a17246f43df890b8e9918401aa7 2024-11-24T16:41:21,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/83426ae4bc5d4cffa86c8abe3a2e7cfd is 50, key is test_row_0/B:col10/1732466478874/Put/seqid=0 2024-11-24T16:41:21,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742038_1214 (size=12301) 2024-11-24T16:41:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T16:41:21,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466541174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466541184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466541383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466541394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,503 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/83426ae4bc5d4cffa86c8abe3a2e7cfd 2024-11-24T16:41:21,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/d4fb0f30eebe4a62821f4f8ac81d6ed6 is 50, key is test_row_0/C:col10/1732466478874/Put/seqid=0 2024-11-24T16:41:21,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742039_1215 (size=12301) 2024-11-24T16:41:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T16:41:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466541703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466541703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:21,916 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/d4fb0f30eebe4a62821f4f8ac81d6ed6 2024-11-24T16:41:21,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/90168a17246f43df890b8e9918401aa7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/90168a17246f43df890b8e9918401aa7 2024-11-24T16:41:21,925 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/90168a17246f43df890b8e9918401aa7, entries=150, sequenceid=391, filesize=30.5 K 2024-11-24T16:41:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/83426ae4bc5d4cffa86c8abe3a2e7cfd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/83426ae4bc5d4cffa86c8abe3a2e7cfd 2024-11-24T16:41:21,931 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/83426ae4bc5d4cffa86c8abe3a2e7cfd, entries=150, sequenceid=391, filesize=12.0 K 2024-11-24T16:41:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/d4fb0f30eebe4a62821f4f8ac81d6ed6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/d4fb0f30eebe4a62821f4f8ac81d6ed6 2024-11-24T16:41:21,936 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/d4fb0f30eebe4a62821f4f8ac81d6ed6, entries=150, sequenceid=391, filesize=12.0 K 2024-11-24T16:41:21,937 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 68db56cd5236f933146f262f370fc8cc in 1278ms, sequenceid=391, compaction requested=true 2024-11-24T16:41:21,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:21,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:21,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-24T16:41:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-24T16:41:21,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T16:41:21,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4310 sec 2024-11-24T16:41:21,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4360 sec 2024-11-24T16:41:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:22,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:41:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:22,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:22,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fc38f6f9119d42fe9f661a2fd58dd15a_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466481067/Put/seqid=0 2024-11-24T16:41:22,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742040_1216 (size=14994) 2024-11-24T16:41:22,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466542068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466542070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466542184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466542184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466542213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466542213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466542394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466542394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,460 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:22,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fc38f6f9119d42fe9f661a2fd58dd15a_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fc38f6f9119d42fe9f661a2fd58dd15a_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:22,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/77a90ae5930c4e688f2a8608eee9c361, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:22,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/77a90ae5930c4e688f2a8608eee9c361 is 175, key is test_row_0/A:col10/1732466481067/Put/seqid=0 2024-11-24T16:41:22,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742041_1217 (size=39949) 2024-11-24T16:41:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T16:41:22,654 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:22,654 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-24T16:41:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T16:41:22,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:22,657 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:22,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466542704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466542704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:22,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T16:41:22,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T16:41:22,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:22,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:22,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:22,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:22,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:22,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/77a90ae5930c4e688f2a8608eee9c361 2024-11-24T16:41:22,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/7190d98b3b434c9a8aa87bfe9b830e4f is 50, key is test_row_0/B:col10/1732466481067/Put/seqid=0 2024-11-24T16:41:22,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742042_1218 (size=12301) 2024-11-24T16:41:22,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T16:41:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:22,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:22,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T16:41:23,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T16:41:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:23,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466543224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466543224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466543233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:23,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466543233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:23,237 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,237 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 68db56cd5236f933146f262f370fc8cc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:23,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T16:41:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T16:41:23,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/7190d98b3b434c9a8aa87bfe9b830e4f 2024-11-24T16:41:23,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/08432ed517d04c019e47096f6bbffe1a is 50, key is test_row_0/C:col10/1732466481067/Put/seqid=0 2024-11-24T16:41:23,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742043_1219 (size=12301) 2024-11-24T16:41:23,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T16:41:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T16:41:23,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:23,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:23,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/08432ed517d04c019e47096f6bbffe1a 2024-11-24T16:41:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/77a90ae5930c4e688f2a8608eee9c361 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/77a90ae5930c4e688f2a8608eee9c361 2024-11-24T16:41:23,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/77a90ae5930c4e688f2a8608eee9c361, entries=200, sequenceid=411, filesize=39.0 K 2024-11-24T16:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/7190d98b3b434c9a8aa87bfe9b830e4f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7190d98b3b434c9a8aa87bfe9b830e4f 2024-11-24T16:41:23,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7190d98b3b434c9a8aa87bfe9b830e4f, entries=150, sequenceid=411, filesize=12.0 K 2024-11-24T16:41:23,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/08432ed517d04c019e47096f6bbffe1a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/08432ed517d04c019e47096f6bbffe1a 2024-11-24T16:41:23,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/08432ed517d04c019e47096f6bbffe1a, entries=150, sequenceid=411, filesize=12.0 K 2024-11-24T16:41:23,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 68db56cd5236f933146f262f370fc8cc in 1673ms, sequenceid=411, compaction requested=true 2024-11-24T16:41:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:23,719 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:23,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:23,719 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:23,720 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:23,720 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:23,721 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/A is initiating minor compaction (all files) 2024-11-24T16:41:23,721 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/B is initiating minor compaction (all files) 2024-11-24T16:41:23,721 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/A in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,721 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/B in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,721 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/71c3bf50ccf341a2be1ceb24953c3f27, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/66a4a112ae29434098e4dda70300047d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/90168a17246f43df890b8e9918401aa7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/77a90ae5930c4e688f2a8608eee9c361] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=131.1 K 2024-11-24T16:41:23,721 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3735d08d1b3e4384a30f127ff5d9f952, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a0f536c904594f2f83dc00429856b811, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/83426ae4bc5d4cffa86c8abe3a2e7cfd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7190d98b3b434c9a8aa87bfe9b830e4f] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=48.6 K 2024-11-24T16:41:23,721 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,721 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/71c3bf50ccf341a2be1ceb24953c3f27, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/66a4a112ae29434098e4dda70300047d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/90168a17246f43df890b8e9918401aa7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/77a90ae5930c4e688f2a8608eee9c361] 2024-11-24T16:41:23,721 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3735d08d1b3e4384a30f127ff5d9f952, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732466476509 2024-11-24T16:41:23,721 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c3bf50ccf341a2be1ceb24953c3f27, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732466476509 2024-11-24T16:41:23,722 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f536c904594f2f83dc00429856b811, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732466477688 2024-11-24T16:41:23,722 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a4a112ae29434098e4dda70300047d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732466477688 2024-11-24T16:41:23,722 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 83426ae4bc5d4cffa86c8abe3a2e7cfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732466478865 2024-11-24T16:41:23,723 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7190d98b3b434c9a8aa87bfe9b830e4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732466481053 2024-11-24T16:41:23,723 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90168a17246f43df890b8e9918401aa7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732466478865 2024-11-24T16:41:23,723 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a90ae5930c4e688f2a8608eee9c361, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732466481053 2024-11-24T16:41:23,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T16:41:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:23,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T16:41:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:23,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:23,734 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:23,734 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:23,735 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/b6efae3e55504bdda3f174e031b9ac8a is 50, key is test_row_0/B:col10/1732466481067/Put/seqid=0 2024-11-24T16:41:23,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b48b268ae464963b05f62654f6c619d_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466482066/Put/seqid=0 2024-11-24T16:41:23,745 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124e3e96418014a4484b83b2782999bb7bc_68db56cd5236f933146f262f370fc8cc store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:23,748 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124e3e96418014a4484b83b2782999bb7bc_68db56cd5236f933146f262f370fc8cc, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:23,748 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e3e96418014a4484b83b2782999bb7bc_68db56cd5236f933146f262f370fc8cc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:23,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742044_1220 (size=13017) 2024-11-24T16:41:23,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742045_1221 (size=12454) 2024-11-24T16:41:23,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742046_1222 (size=4469) 2024-11-24T16:41:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T16:41:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:24,163 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:24,164 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/719218d32f9244ceaa35aa35a889e37e is 175, key is test_row_0/A:col10/1732466481067/Put/seqid=0 2024-11-24T16:41:24,168 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/b6efae3e55504bdda3f174e031b9ac8a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/b6efae3e55504bdda3f174e031b9ac8a 2024-11-24T16:41:24,169 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b48b268ae464963b05f62654f6c619d_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b48b268ae464963b05f62654f6c619d_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:24,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/831087a7ed1444b0ae182cc335e92ebf, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:24,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/831087a7ed1444b0ae182cc335e92ebf is 175, key is test_row_0/A:col10/1732466482066/Put/seqid=0 2024-11-24T16:41:24,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742047_1223 (size=31971) 2024-11-24T16:41:24,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742048_1224 (size=31255) 2024-11-24T16:41:24,180 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/719218d32f9244ceaa35aa35a889e37e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/719218d32f9244ceaa35aa35a889e37e 2024-11-24T16:41:24,182 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/B of 68db56cd5236f933146f262f370fc8cc into b6efae3e55504bdda3f174e031b9ac8a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:24,182 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:24,182 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/B, priority=12, startTime=1732466483719; duration=0sec 2024-11-24T16:41:24,182 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:24,183 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:B 2024-11-24T16:41:24,183 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:24,185 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:24,185 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/C is initiating minor compaction (all files) 2024-11-24T16:41:24,185 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/C in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:24,185 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c2d7ceb8b1b480ca327135dadf26edf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e2706aba6ead4915a18f93d40b366d75, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/d4fb0f30eebe4a62821f4f8ac81d6ed6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/08432ed517d04c019e47096f6bbffe1a] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=48.6 K 2024-11-24T16:41:24,186 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c2d7ceb8b1b480ca327135dadf26edf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732466476509 2024-11-24T16:41:24,187 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e2706aba6ead4915a18f93d40b366d75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732466477688 2024-11-24T16:41:24,187 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/A of 68db56cd5236f933146f262f370fc8cc into 719218d32f9244ceaa35aa35a889e37e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:24,187 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:24,187 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/A, priority=12, startTime=1732466483719; duration=0sec 2024-11-24T16:41:24,187 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:24,187 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:A 2024-11-24T16:41:24,187 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d4fb0f30eebe4a62821f4f8ac81d6ed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732466478865 2024-11-24T16:41:24,188 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 08432ed517d04c019e47096f6bbffe1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732466481053 2024-11-24T16:41:24,198 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#C#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:24,199 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/c619ef9bcb024013b45e394d6220dd1f is 50, key is test_row_0/C:col10/1732466481067/Put/seqid=0 2024-11-24T16:41:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742049_1225 (size=13017) 2024-11-24T16:41:24,208 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/c619ef9bcb024013b45e394d6220dd1f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/c619ef9bcb024013b45e394d6220dd1f 2024-11-24T16:41:24,216 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/C of 68db56cd5236f933146f262f370fc8cc into c619ef9bcb024013b45e394d6220dd1f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:24,216 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:24,216 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/C, priority=12, startTime=1732466483719; duration=0sec 2024-11-24T16:41:24,216 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:24,216 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:C 2024-11-24T16:41:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:24,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. as already flushing 2024-11-24T16:41:24,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466484260, max timestamp=1732466484291Num uniques 859; 2024-11-24T16:41:24,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466484260, max timestamp=1732466484291Num uniques 859; 2024-11-24T16:41:24,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:41:24,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:41:24,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466484260, max timestamp=1732466484291Num uniques 859; 2024-11-24T16:41:24,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:41:24,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466544349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466544349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:24,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466544463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466544463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:24,581 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/831087a7ed1444b0ae182cc335e92ebf 2024-11-24T16:41:24,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/a53c2c1b24c84974bb85d9e00f7f8c17 is 50, key is test_row_0/B:col10/1732466482066/Put/seqid=0 2024-11-24T16:41:24,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742050_1226 (size=12301) 2024-11-24T16:41:24,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466544674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466544674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:24,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T16:41:24,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466544984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:24,993 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/a53c2c1b24c84974bb85d9e00f7f8c17 2024-11-24T16:41:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466544984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/40856abdd49547cdbe4032508dd50110 is 50, key is test_row_0/C:col10/1732466482066/Put/seqid=0 2024-11-24T16:41:25,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742051_1227 (size=12301) 2024-11-24T16:41:25,022 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/40856abdd49547cdbe4032508dd50110 2024-11-24T16:41:25,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/831087a7ed1444b0ae182cc335e92ebf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/831087a7ed1444b0ae182cc335e92ebf 2024-11-24T16:41:25,032 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/831087a7ed1444b0ae182cc335e92ebf, entries=150, sequenceid=423, filesize=30.5 K 2024-11-24T16:41:25,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/a53c2c1b24c84974bb85d9e00f7f8c17 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a53c2c1b24c84974bb85d9e00f7f8c17 2024-11-24T16:41:25,039 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a53c2c1b24c84974bb85d9e00f7f8c17, entries=150, sequenceid=423, filesize=12.0 K 2024-11-24T16:41:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/40856abdd49547cdbe4032508dd50110 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/40856abdd49547cdbe4032508dd50110 2024-11-24T16:41:25,046 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/40856abdd49547cdbe4032508dd50110, entries=150, sequenceid=423, filesize=12.0 K 2024-11-24T16:41:25,048 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 68db56cd5236f933146f262f370fc8cc in 1319ms, sequenceid=423, compaction requested=false 2024-11-24T16:41:25,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:25,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:25,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-24T16:41:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-24T16:41:25,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T16:41:25,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3920 sec 2024-11-24T16:41:25,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.3970 sec 2024-11-24T16:41:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:25,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-24T16:41:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:25,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:25,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dcea88f3d51442f99afe44de92ba134a_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466484347/Put/seqid=0 2024-11-24T16:41:25,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742052_1228 (size=14994) 2024-11-24T16:41:25,275 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:25,279 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dcea88f3d51442f99afe44de92ba134a_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dcea88f3d51442f99afe44de92ba134a_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:25,280 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/0641e87354b14dbea57124ff7c7649f9, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:25,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/0641e87354b14dbea57124ff7c7649f9 is 175, key is test_row_0/A:col10/1732466484347/Put/seqid=0 2024-11-24T16:41:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466545277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466545283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742053_1229 (size=39949) 2024-11-24T16:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,288 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=460, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/0641e87354b14dbea57124ff7c7649f9 2024-11-24T16:41:25,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/272d3726e21e47c592609deb2c205735 is 50, key is test_row_0/B:col10/1732466484347/Put/seqid=0 2024-11-24T16:41:25,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742054_1230 (size=12301) 2024-11-24T16:41:25,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466545394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466545394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58232 deadline: 1732466545503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58264 deadline: 1732466545503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466545603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466545603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/272d3726e21e47c592609deb2c205735 2024-11-24T16:41:25,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/6c10ab032db245f183e4d3f948d59dca is 50, key is test_row_0/C:col10/1732466484347/Put/seqid=0 2024-11-24T16:41:25,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742055_1231 (size=12301) 2024-11-24T16:41:25,724 INFO [Thread-685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-685. 2024-11-24T16:41:25,724 DEBUG [Thread-685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:25,724 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,724 DEBUG [Thread-685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:25,726 INFO [Thread-687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-687. 2024-11-24T16:41:25,726 DEBUG [Thread-687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:25,726 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,726 DEBUG [Thread-687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:25,728 INFO [Thread-689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-689. 2024-11-24T16:41:25,728 DEBUG [Thread-689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:25,728 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,728 DEBUG [Thread-689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:25,729 INFO [Thread-683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-683. 2024-11-24T16:41:25,729 DEBUG [Thread-683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:25,729 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,729 DEBUG [Thread-683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:25,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58276 deadline: 1732466545914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58238 deadline: 1732466545913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:41:26,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/6c10ab032db245f183e4d3f948d59dca 2024-11-24T16:41:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/0641e87354b14dbea57124ff7c7649f9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0641e87354b14dbea57124ff7c7649f9 2024-11-24T16:41:26,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0641e87354b14dbea57124ff7c7649f9, entries=200, sequenceid=460, filesize=39.0 K 2024-11-24T16:41:26,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/272d3726e21e47c592609deb2c205735 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/272d3726e21e47c592609deb2c205735 2024-11-24T16:41:26,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/272d3726e21e47c592609deb2c205735, entries=150, sequenceid=460, filesize=12.0 K 2024-11-24T16:41:26,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/6c10ab032db245f183e4d3f948d59dca as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/6c10ab032db245f183e4d3f948d59dca 2024-11-24T16:41:26,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/6c10ab032db245f183e4d3f948d59dca, entries=150, sequenceid=460, filesize=12.0 K 2024-11-24T16:41:26,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 68db56cd5236f933146f262f370fc8cc in 882ms, sequenceid=460, compaction requested=true 2024-11-24T16:41:26,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:26,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:26,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:26,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 68db56cd5236f933146f262f370fc8cc:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:26,139 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:26,139 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:26,139 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:26,139 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:26,139 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/B is initiating minor compaction (all files) 2024-11-24T16:41:26,140 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/B in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:26,140 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/A is initiating minor compaction (all files) 2024-11-24T16:41:26,140 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/b6efae3e55504bdda3f174e031b9ac8a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a53c2c1b24c84974bb85d9e00f7f8c17, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/272d3726e21e47c592609deb2c205735] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=36.7 K 2024-11-24T16:41:26,140 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/A in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:26,140 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/719218d32f9244ceaa35aa35a889e37e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/831087a7ed1444b0ae182cc335e92ebf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0641e87354b14dbea57124ff7c7649f9] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=100.8 K 2024-11-24T16:41:26,140 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:26,140 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/719218d32f9244ceaa35aa35a889e37e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/831087a7ed1444b0ae182cc335e92ebf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0641e87354b14dbea57124ff7c7649f9] 2024-11-24T16:41:26,140 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting b6efae3e55504bdda3f174e031b9ac8a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732466481053 2024-11-24T16:41:26,140 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a53c2c1b24c84974bb85d9e00f7f8c17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732466482062 2024-11-24T16:41:26,140 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719218d32f9244ceaa35aa35a889e37e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732466481053 2024-11-24T16:41:26,141 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 272d3726e21e47c592609deb2c205735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732466484347 2024-11-24T16:41:26,141 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831087a7ed1444b0ae182cc335e92ebf, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732466482062 2024-11-24T16:41:26,141 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0641e87354b14dbea57124ff7c7649f9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732466484328 2024-11-24T16:41:26,149 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:26,149 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/86d6428b49e748e5a8192e1dca5ad224 is 50, key is test_row_0/B:col10/1732466484347/Put/seqid=0 2024-11-24T16:41:26,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742056_1232 (size=13119) 2024-11-24T16:41:26,155 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:26,157 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411243a8fc2aae45f4c9298cdcfcce5465fa8_68db56cd5236f933146f262f370fc8cc store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:26,174 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411243a8fc2aae45f4c9298cdcfcce5465fa8_68db56cd5236f933146f262f370fc8cc, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:26,174 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243a8fc2aae45f4c9298cdcfcce5465fa8_68db56cd5236f933146f262f370fc8cc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:26,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742057_1233 (size=4469) 2024-11-24T16:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:26,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:41:26,425 INFO [Thread-680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-680. 2024-11-24T16:41:26,425 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-24T16:41:26,425 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:26,425 DEBUG [Thread-680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:26,425 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,425 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:26,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:26,425 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,425 DEBUG [Thread-680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:26,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:26,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:26,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:26,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:26,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:26,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112482b48ab690484fe18edae4be10cc9998_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466486424/Put/seqid=0 2024-11-24T16:41:26,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742058_1234 (size=12454) 2024-11-24T16:41:26,515 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-24T16:41:26,515 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:26,515 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,516 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:26,524 INFO [Thread-678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-678. 2024-11-24T16:41:26,524 DEBUG [Thread-678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:26,524 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,524 DEBUG [Thread-678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:26,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:26,558 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/86d6428b49e748e5a8192e1dca5ad224 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/86d6428b49e748e5a8192e1dca5ad224 2024-11-24T16:41:26,563 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/B of 68db56cd5236f933146f262f370fc8cc into 86d6428b49e748e5a8192e1dca5ad224(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:26,563 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:26,563 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/B, priority=13, startTime=1732466486138; duration=0sec 2024-11-24T16:41:26,563 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:26,563 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:B 2024-11-24T16:41:26,563 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:26,564 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:26,564 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 68db56cd5236f933146f262f370fc8cc/C is initiating minor compaction (all files) 2024-11-24T16:41:26,564 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 68db56cd5236f933146f262f370fc8cc/C in TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:26,564 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/c619ef9bcb024013b45e394d6220dd1f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/40856abdd49547cdbe4032508dd50110, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/6c10ab032db245f183e4d3f948d59dca] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp, totalSize=36.7 K 2024-11-24T16:41:26,565 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c619ef9bcb024013b45e394d6220dd1f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732466481053 2024-11-24T16:41:26,565 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 40856abdd49547cdbe4032508dd50110, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1732466482062 2024-11-24T16:41:26,565 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c10ab032db245f183e4d3f948d59dca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732466484347 2024-11-24T16:41:26,575 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#C#compaction#195 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:26,575 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0fd024efdc96406f8cb2b1fb15b2609f is 50, key is test_row_0/C:col10/1732466484347/Put/seqid=0 2024-11-24T16:41:26,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742059_1235 (size=13119) 2024-11-24T16:41:26,579 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 68db56cd5236f933146f262f370fc8cc#A#compaction#193 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:26,580 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/9c06ecaac46f4d6188801ac45dc36e9a is 175, key is test_row_0/A:col10/1732466484347/Put/seqid=0 2024-11-24T16:41:26,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742060_1236 (size=32073) 2024-11-24T16:41:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T16:41:26,814 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:26,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:26,840 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112482b48ab690484fe18edae4be10cc9998_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112482b48ab690484fe18edae4be10cc9998_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:26,841 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/631ce7b389a4420cb1218e13048e85b6, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:26,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/631ce7b389a4420cb1218e13048e85b6 is 175, key is test_row_0/A:col10/1732466486424/Put/seqid=0 2024-11-24T16:41:26,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742061_1237 (size=31255) 2024-11-24T16:41:26,984 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/0fd024efdc96406f8cb2b1fb15b2609f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0fd024efdc96406f8cb2b1fb15b2609f 2024-11-24T16:41:26,988 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/9c06ecaac46f4d6188801ac45dc36e9a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/9c06ecaac46f4d6188801ac45dc36e9a 2024-11-24T16:41:26,989 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/C of 68db56cd5236f933146f262f370fc8cc into 0fd024efdc96406f8cb2b1fb15b2609f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:26,989 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:26,989 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/C, priority=13, startTime=1732466486139; duration=0sec 2024-11-24T16:41:26,989 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:26,989 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:C 2024-11-24T16:41:26,993 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 68db56cd5236f933146f262f370fc8cc/A of 68db56cd5236f933146f262f370fc8cc into 9c06ecaac46f4d6188801ac45dc36e9a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:26,993 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:26,993 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., storeName=68db56cd5236f933146f262f370fc8cc/A, priority=13, startTime=1732466486138; duration=0sec 2024-11-24T16:41:26,993 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:26,993 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 68db56cd5236f933146f262f370fc8cc:A 2024-11-24T16:41:27,246 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=473, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/631ce7b389a4420cb1218e13048e85b6 2024-11-24T16:41:27,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/22bdd6a41aff4ea4b64068f61d1e9741 is 50, key is test_row_0/B:col10/1732466486424/Put/seqid=0 2024-11-24T16:41:27,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742062_1238 (size=12301) 2024-11-24T16:41:27,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/22bdd6a41aff4ea4b64068f61d1e9741 2024-11-24T16:41:27,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/4402f177d59346a2bdde3474b5860c01 is 50, key is test_row_0/C:col10/1732466486424/Put/seqid=0 2024-11-24T16:41:27,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742063_1239 (size=12301) 2024-11-24T16:41:27,915 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-24T16:41:27,915 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:27,915 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:27,915 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:27,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2527 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2522 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3303 rows 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1106 2024-11-24T16:41:27,915 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3318 rows 2024-11-24T16:41:27,915 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:41:27,915 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:41:27,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:27,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:27,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:27,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T16:41:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T16:41:27,919 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466487918"}]},"ts":"1732466487918"} 2024-11-24T16:41:27,920 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T16:41:27,920 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T16:41:27,921 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:41:27,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, UNASSIGN}] 2024-11-24T16:41:27,922 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, UNASSIGN 2024-11-24T16:41:27,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:27,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, UNASSIGN because future has completed 2024-11-24T16:41:27,924 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:41:27,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T16:41:28,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/4402f177d59346a2bdde3474b5860c01 2024-11-24T16:41:28,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/631ce7b389a4420cb1218e13048e85b6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/631ce7b389a4420cb1218e13048e85b6 2024-11-24T16:41:28,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/631ce7b389a4420cb1218e13048e85b6, entries=150, sequenceid=473, filesize=30.5 K 2024-11-24T16:41:28,076 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:28,076 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:41:28,076 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 68db56cd5236f933146f262f370fc8cc, disabling compactions & flushes 2024-11-24T16:41:28,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/22bdd6a41aff4ea4b64068f61d1e9741 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/22bdd6a41aff4ea4b64068f61d1e9741 2024-11-24T16:41:28,077 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:28,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/22bdd6a41aff4ea4b64068f61d1e9741, entries=150, sequenceid=473, filesize=12.0 K 2024-11-24T16:41:28,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/4402f177d59346a2bdde3474b5860c01 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/4402f177d59346a2bdde3474b5860c01 2024-11-24T16:41:28,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/4402f177d59346a2bdde3474b5860c01, entries=150, sequenceid=473, filesize=12.0 K 2024-11-24T16:41:28,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for 68db56cd5236f933146f262f370fc8cc in 1659ms, sequenceid=473, compaction requested=false 2024-11-24T16:41:28,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 68db56cd5236f933146f262f370fc8cc: 2024-11-24T16:41:28,084 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:28,084 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:28,084 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. after waiting 0 ms 2024-11-24T16:41:28,084 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:28,085 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 68db56cd5236f933146f262f370fc8cc 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T16:41:28,085 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=A 2024-11-24T16:41:28,085 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:28,085 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=B 2024-11-24T16:41:28,085 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:28,085 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 68db56cd5236f933146f262f370fc8cc, store=C 2024-11-24T16:41:28,085 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:28,090 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241bbe6f5701404181aadb31aef2e763e3_68db56cd5236f933146f262f370fc8cc is 50, key is test_row_0/A:col10/1732466486514/Put/seqid=0 2024-11-24T16:41:28,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742064_1240 (size=12454) 2024-11-24T16:41:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T16:41:28,494 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:28,498 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241bbe6f5701404181aadb31aef2e763e3_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241bbe6f5701404181aadb31aef2e763e3_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:28,499 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4695201240784a7ab34cbb9e973b855d, store: [table=TestAcidGuarantees family=A region=68db56cd5236f933146f262f370fc8cc] 2024-11-24T16:41:28,499 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4695201240784a7ab34cbb9e973b855d is 175, key is test_row_0/A:col10/1732466486514/Put/seqid=0 2024-11-24T16:41:28,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742065_1241 (size=31255) 2024-11-24T16:41:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T16:41:28,903 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=482, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4695201240784a7ab34cbb9e973b855d 2024-11-24T16:41:28,909 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/ff00e5e87126416ba6ee26e6c4cc6300 is 50, key is test_row_0/B:col10/1732466486514/Put/seqid=0 2024-11-24T16:41:28,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742066_1242 (size=12301) 2024-11-24T16:41:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T16:41:29,313 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/ff00e5e87126416ba6ee26e6c4cc6300 2024-11-24T16:41:29,320 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/1ffb3f160d4e4162be139c9b53346260 is 50, key is test_row_0/C:col10/1732466486514/Put/seqid=0 2024-11-24T16:41:29,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742067_1243 (size=12301) 2024-11-24T16:41:29,335 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T16:41:29,724 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/1ffb3f160d4e4162be139c9b53346260 2024-11-24T16:41:29,728 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/A/4695201240784a7ab34cbb9e973b855d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4695201240784a7ab34cbb9e973b855d 2024-11-24T16:41:29,732 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4695201240784a7ab34cbb9e973b855d, entries=150, sequenceid=482, filesize=30.5 K 2024-11-24T16:41:29,733 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/B/ff00e5e87126416ba6ee26e6c4cc6300 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/ff00e5e87126416ba6ee26e6c4cc6300 2024-11-24T16:41:29,736 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/ff00e5e87126416ba6ee26e6c4cc6300, entries=150, sequenceid=482, filesize=12.0 K 2024-11-24T16:41:29,737 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/.tmp/C/1ffb3f160d4e4162be139c9b53346260 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1ffb3f160d4e4162be139c9b53346260 2024-11-24T16:41:29,740 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1ffb3f160d4e4162be139c9b53346260, entries=150, sequenceid=482, filesize=12.0 K 2024-11-24T16:41:29,741 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 68db56cd5236f933146f262f370fc8cc in 1657ms, sequenceid=482, compaction requested=true 2024-11-24T16:41:29,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/56217b40019a4cc9a65e4abfcbca1dd4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/e2d389fcf68a46bab6e5f6b47b8615c5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/aa90d7e9a7eb4dd3bdabe2053ea5513b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/83677f7936d74c2ab493381e9055d4d7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/73a4fdf0d8f8491ea30d16d5e897f8cf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4b68a5506fc549debb9724979446772f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/1a19acfa805f49fabc2106c9b1c9b29f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0a82235b7ee14fa2949cc140ea8df1eb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3b132d8ed73f4ee4a99f6fd386e6e9c0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7ff823fead2140348459905df882779a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/17cb0dd1dd3a4edfabc5c182db530143, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3388c34bb8cb426594187683f8136304, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7edc932681f546d8b476a3801e89c0c7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/f92fc4ca826d40c98c4f2347265b941d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/152eae1619d64c618f3bc3add22f544e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/d191a838ba644805b7cf1e1c1a86096e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/8c0b1f62a5a348269bf4fc4801d64a01, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/71c3bf50ccf341a2be1ceb24953c3f27, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/66a4a112ae29434098e4dda70300047d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/90168a17246f43df890b8e9918401aa7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/77a90ae5930c4e688f2a8608eee9c361, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/719218d32f9244ceaa35aa35a889e37e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/831087a7ed1444b0ae182cc335e92ebf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0641e87354b14dbea57124ff7c7649f9] to archive 2024-11-24T16:41:29,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:29,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/56217b40019a4cc9a65e4abfcbca1dd4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/56217b40019a4cc9a65e4abfcbca1dd4 2024-11-24T16:41:29,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/e2d389fcf68a46bab6e5f6b47b8615c5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/e2d389fcf68a46bab6e5f6b47b8615c5 2024-11-24T16:41:29,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/aa90d7e9a7eb4dd3bdabe2053ea5513b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/aa90d7e9a7eb4dd3bdabe2053ea5513b 2024-11-24T16:41:29,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/83677f7936d74c2ab493381e9055d4d7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/83677f7936d74c2ab493381e9055d4d7 2024-11-24T16:41:29,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/73a4fdf0d8f8491ea30d16d5e897f8cf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/73a4fdf0d8f8491ea30d16d5e897f8cf 2024-11-24T16:41:29,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4b68a5506fc549debb9724979446772f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4b68a5506fc549debb9724979446772f 2024-11-24T16:41:29,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/1a19acfa805f49fabc2106c9b1c9b29f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/1a19acfa805f49fabc2106c9b1c9b29f 2024-11-24T16:41:29,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0a82235b7ee14fa2949cc140ea8df1eb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0a82235b7ee14fa2949cc140ea8df1eb 2024-11-24T16:41:29,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3b132d8ed73f4ee4a99f6fd386e6e9c0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3b132d8ed73f4ee4a99f6fd386e6e9c0 2024-11-24T16:41:29,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7ff823fead2140348459905df882779a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7ff823fead2140348459905df882779a 2024-11-24T16:41:29,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/17cb0dd1dd3a4edfabc5c182db530143 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/17cb0dd1dd3a4edfabc5c182db530143 2024-11-24T16:41:29,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3388c34bb8cb426594187683f8136304 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/3388c34bb8cb426594187683f8136304 2024-11-24T16:41:29,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7edc932681f546d8b476a3801e89c0c7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/7edc932681f546d8b476a3801e89c0c7 2024-11-24T16:41:29,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/f92fc4ca826d40c98c4f2347265b941d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/f92fc4ca826d40c98c4f2347265b941d 2024-11-24T16:41:29,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/152eae1619d64c618f3bc3add22f544e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/152eae1619d64c618f3bc3add22f544e 2024-11-24T16:41:29,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/d191a838ba644805b7cf1e1c1a86096e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/d191a838ba644805b7cf1e1c1a86096e 2024-11-24T16:41:29,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/8c0b1f62a5a348269bf4fc4801d64a01 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/8c0b1f62a5a348269bf4fc4801d64a01 2024-11-24T16:41:29,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/71c3bf50ccf341a2be1ceb24953c3f27 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/71c3bf50ccf341a2be1ceb24953c3f27 2024-11-24T16:41:29,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/66a4a112ae29434098e4dda70300047d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/66a4a112ae29434098e4dda70300047d 2024-11-24T16:41:29,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/90168a17246f43df890b8e9918401aa7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/90168a17246f43df890b8e9918401aa7 2024-11-24T16:41:29,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/77a90ae5930c4e688f2a8608eee9c361 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/77a90ae5930c4e688f2a8608eee9c361 2024-11-24T16:41:29,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/719218d32f9244ceaa35aa35a889e37e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/719218d32f9244ceaa35aa35a889e37e 2024-11-24T16:41:29,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/831087a7ed1444b0ae182cc335e92ebf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/831087a7ed1444b0ae182cc335e92ebf 2024-11-24T16:41:29,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0641e87354b14dbea57124ff7c7649f9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/0641e87354b14dbea57124ff7c7649f9 2024-11-24T16:41:29,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/149e1b8e4bd74751be8d82d61a15420d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/e59789c06e65495587b6c2696fe9ef2a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/bd7f00baa91d4ba793c2ac198d6c31b4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/00046ac72a584a66bc100c05c3348e84, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/64ba858726224741826fa0d9c2d2f62f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/53cca20710924d81a46fca251cea450c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aadb130699174f129fe4188cc1f8eea7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7076bdc31e6d443d981d9e5df42b13c8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/695b79de95434fafb6d82e3c07ef25da, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aa9f9d4c256b462cb53a7848289095ab, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/07b16ee8322a40119eb99c0fe1affe29, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f7265bb169f6412a893a9f044355ec76, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/eb171312039747caab8bab6ea3349013, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f046654da2654f0fbbadf635708fe51e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/19d605272154439ebdf366891d82542e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3ac76cd353db42738815a7d6d0c64365, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3735d08d1b3e4384a30f127ff5d9f952, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/9586da7a1dc94e6ca63dc24de4a021bb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a0f536c904594f2f83dc00429856b811, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/83426ae4bc5d4cffa86c8abe3a2e7cfd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/b6efae3e55504bdda3f174e031b9ac8a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7190d98b3b434c9a8aa87bfe9b830e4f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a53c2c1b24c84974bb85d9e00f7f8c17, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/272d3726e21e47c592609deb2c205735] to archive 2024-11-24T16:41:29,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:29,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/149e1b8e4bd74751be8d82d61a15420d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/149e1b8e4bd74751be8d82d61a15420d 2024-11-24T16:41:29,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/e59789c06e65495587b6c2696fe9ef2a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/e59789c06e65495587b6c2696fe9ef2a 2024-11-24T16:41:29,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/bd7f00baa91d4ba793c2ac198d6c31b4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/bd7f00baa91d4ba793c2ac198d6c31b4 2024-11-24T16:41:29,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/00046ac72a584a66bc100c05c3348e84 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/00046ac72a584a66bc100c05c3348e84 2024-11-24T16:41:29,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/64ba858726224741826fa0d9c2d2f62f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/64ba858726224741826fa0d9c2d2f62f 2024-11-24T16:41:29,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/53cca20710924d81a46fca251cea450c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/53cca20710924d81a46fca251cea450c 2024-11-24T16:41:29,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aadb130699174f129fe4188cc1f8eea7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aadb130699174f129fe4188cc1f8eea7 2024-11-24T16:41:29,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7076bdc31e6d443d981d9e5df42b13c8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7076bdc31e6d443d981d9e5df42b13c8 2024-11-24T16:41:29,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/695b79de95434fafb6d82e3c07ef25da to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/695b79de95434fafb6d82e3c07ef25da 2024-11-24T16:41:29,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aa9f9d4c256b462cb53a7848289095ab to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/aa9f9d4c256b462cb53a7848289095ab 2024-11-24T16:41:29,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/07b16ee8322a40119eb99c0fe1affe29 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/07b16ee8322a40119eb99c0fe1affe29 2024-11-24T16:41:29,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f7265bb169f6412a893a9f044355ec76 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f7265bb169f6412a893a9f044355ec76 2024-11-24T16:41:29,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/eb171312039747caab8bab6ea3349013 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/eb171312039747caab8bab6ea3349013 2024-11-24T16:41:29,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f046654da2654f0fbbadf635708fe51e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/f046654da2654f0fbbadf635708fe51e 2024-11-24T16:41:29,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/19d605272154439ebdf366891d82542e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/19d605272154439ebdf366891d82542e 2024-11-24T16:41:29,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3ac76cd353db42738815a7d6d0c64365 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3ac76cd353db42738815a7d6d0c64365 2024-11-24T16:41:29,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3735d08d1b3e4384a30f127ff5d9f952 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/3735d08d1b3e4384a30f127ff5d9f952 2024-11-24T16:41:29,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/9586da7a1dc94e6ca63dc24de4a021bb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/9586da7a1dc94e6ca63dc24de4a021bb 2024-11-24T16:41:29,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a0f536c904594f2f83dc00429856b811 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a0f536c904594f2f83dc00429856b811 2024-11-24T16:41:29,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/83426ae4bc5d4cffa86c8abe3a2e7cfd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/83426ae4bc5d4cffa86c8abe3a2e7cfd 2024-11-24T16:41:29,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/b6efae3e55504bdda3f174e031b9ac8a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/b6efae3e55504bdda3f174e031b9ac8a 2024-11-24T16:41:29,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7190d98b3b434c9a8aa87bfe9b830e4f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/7190d98b3b434c9a8aa87bfe9b830e4f 2024-11-24T16:41:29,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a53c2c1b24c84974bb85d9e00f7f8c17 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/a53c2c1b24c84974bb85d9e00f7f8c17 2024-11-24T16:41:29,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/272d3726e21e47c592609deb2c205735 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/272d3726e21e47c592609deb2c205735 2024-11-24T16:41:29,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/194806cf526b448a9d9f7cb3a6e0cd2c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1b4e04a755d844878265fce3d5a3fda5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/05dcaeab001745649cec779a6a020725, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fbee557b0c624fd38e9d4e94bc6e21c9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/84d61fb2a23e41bf80f3f35ffac55d05, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/9a0a5b782a304ac1ae0036e5d1f45bc3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fb5616d989424b44af8351ba5ddb034f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ff63390deafd4de4b36298c3192a9460, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e7e9591d0112430db343203ca2972389, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f8917d18da6e46138d6e9d0f1cbb510c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0bae676eadcf443abadb76360097b957, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ddde9ad3fd31425ebbba41c5f6f7e24d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/22a498b65f504e81aa44f82628460b18, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/cd4ab22f79c94742ab749f21fb8448e3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c6c62c01dd641a3b9d72ee16ea66cb0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f027f026970a4015803d7490822d0773, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c2d7ceb8b1b480ca327135dadf26edf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0513afdeae0949f9974291a6bd5bdc85, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e2706aba6ead4915a18f93d40b366d75, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/d4fb0f30eebe4a62821f4f8ac81d6ed6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/c619ef9bcb024013b45e394d6220dd1f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/08432ed517d04c019e47096f6bbffe1a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/40856abdd49547cdbe4032508dd50110, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/6c10ab032db245f183e4d3f948d59dca] to archive 2024-11-24T16:41:29,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:29,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/194806cf526b448a9d9f7cb3a6e0cd2c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/194806cf526b448a9d9f7cb3a6e0cd2c 2024-11-24T16:41:29,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1b4e04a755d844878265fce3d5a3fda5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1b4e04a755d844878265fce3d5a3fda5 2024-11-24T16:41:29,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/05dcaeab001745649cec779a6a020725 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/05dcaeab001745649cec779a6a020725 2024-11-24T16:41:29,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fbee557b0c624fd38e9d4e94bc6e21c9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fbee557b0c624fd38e9d4e94bc6e21c9 2024-11-24T16:41:29,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/84d61fb2a23e41bf80f3f35ffac55d05 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/84d61fb2a23e41bf80f3f35ffac55d05 2024-11-24T16:41:29,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/9a0a5b782a304ac1ae0036e5d1f45bc3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/9a0a5b782a304ac1ae0036e5d1f45bc3 2024-11-24T16:41:29,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fb5616d989424b44af8351ba5ddb034f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/fb5616d989424b44af8351ba5ddb034f 2024-11-24T16:41:29,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ff63390deafd4de4b36298c3192a9460 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ff63390deafd4de4b36298c3192a9460 2024-11-24T16:41:29,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e7e9591d0112430db343203ca2972389 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e7e9591d0112430db343203ca2972389 2024-11-24T16:41:29,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f8917d18da6e46138d6e9d0f1cbb510c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f8917d18da6e46138d6e9d0f1cbb510c 2024-11-24T16:41:29,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0bae676eadcf443abadb76360097b957 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0bae676eadcf443abadb76360097b957 2024-11-24T16:41:29,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ddde9ad3fd31425ebbba41c5f6f7e24d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/ddde9ad3fd31425ebbba41c5f6f7e24d 2024-11-24T16:41:29,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/22a498b65f504e81aa44f82628460b18 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/22a498b65f504e81aa44f82628460b18 2024-11-24T16:41:29,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/cd4ab22f79c94742ab749f21fb8448e3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/cd4ab22f79c94742ab749f21fb8448e3 2024-11-24T16:41:29,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c6c62c01dd641a3b9d72ee16ea66cb0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c6c62c01dd641a3b9d72ee16ea66cb0 2024-11-24T16:41:29,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f027f026970a4015803d7490822d0773 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/f027f026970a4015803d7490822d0773 2024-11-24T16:41:29,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c2d7ceb8b1b480ca327135dadf26edf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0c2d7ceb8b1b480ca327135dadf26edf 2024-11-24T16:41:29,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0513afdeae0949f9974291a6bd5bdc85 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0513afdeae0949f9974291a6bd5bdc85 2024-11-24T16:41:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e2706aba6ead4915a18f93d40b366d75 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/e2706aba6ead4915a18f93d40b366d75 2024-11-24T16:41:29,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/d4fb0f30eebe4a62821f4f8ac81d6ed6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/d4fb0f30eebe4a62821f4f8ac81d6ed6 2024-11-24T16:41:29,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/c619ef9bcb024013b45e394d6220dd1f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/c619ef9bcb024013b45e394d6220dd1f 2024-11-24T16:41:29,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/08432ed517d04c019e47096f6bbffe1a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/08432ed517d04c019e47096f6bbffe1a 2024-11-24T16:41:29,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/40856abdd49547cdbe4032508dd50110 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/40856abdd49547cdbe4032508dd50110 2024-11-24T16:41:29,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/6c10ab032db245f183e4d3f948d59dca to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/6c10ab032db245f183e4d3f948d59dca 2024-11-24T16:41:29,825 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/recovered.edits/485.seqid, newMaxSeqId=485, maxSeqId=4 2024-11-24T16:41:29,826 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. 2024-11-24T16:41:29,826 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 68db56cd5236f933146f262f370fc8cc: Waiting for close lock at 1732466488076Running coprocessor pre-close hooks at 1732466488076Disabling compacts and flushes for region at 1732466488076Disabling writes for close at 1732466488084 (+8 ms)Obtaining lock to block concurrent updates at 1732466488085 (+1 ms)Preparing flush snapshotting stores in 68db56cd5236f933146f262f370fc8cc at 1732466488085Finished memstore snapshotting TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732466488085Flushing stores of TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc. at 1732466488085Flushing 68db56cd5236f933146f262f370fc8cc/A: creating writer at 1732466488085Flushing mob file 68db56cd5236f933146f262f370fc8cc/A: appending metadata at 1732466488090 (+5 ms)Flushing mob file 68db56cd5236f933146f262f370fc8cc/A: closing flushed file at 1732466488090Flushing 68db56cd5236f933146f262f370fc8cc/A: appending metadata at 1732466488499 (+409 ms)Flushing 68db56cd5236f933146f262f370fc8cc/A: closing flushed file at 1732466488499Flushing 68db56cd5236f933146f262f370fc8cc/B: creating writer at 1732466488907 (+408 ms)Flushing 68db56cd5236f933146f262f370fc8cc/B: appending metadata at 1732466488909 (+2 ms)Flushing 68db56cd5236f933146f262f370fc8cc/B: closing flushed file at 1732466488909Flushing 68db56cd5236f933146f262f370fc8cc/C: creating writer at 1732466489317 (+408 ms)Flushing 68db56cd5236f933146f262f370fc8cc/C: appending metadata at 1732466489319 (+2 ms)Flushing 68db56cd5236f933146f262f370fc8cc/C: closing flushed file at 1732466489319Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a17fab9: reopening flushed file at 1732466489728 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10a8b7a: reopening flushed file at 1732466489732 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52b41426: reopening flushed file at 1732466489736 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 68db56cd5236f933146f262f370fc8cc in 1657ms, sequenceid=482, compaction requested=true at 1732466489741 (+5 ms)Writing region close event to WAL at 1732466489822 (+81 ms)Running coprocessor post-close hooks at 1732466489826 (+4 ms)Closed at 1732466489826 2024-11-24T16:41:29,828 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:29,829 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=68db56cd5236f933146f262f370fc8cc, regionState=CLOSED 2024-11-24T16:41:29,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:29,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-24T16:41:29,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 68db56cd5236f933146f262f370fc8cc, server=e335232f7491,33607,1732466431346 in 1.9070 sec 2024-11-24T16:41:29,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T16:41:29,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=68db56cd5236f933146f262f370fc8cc, UNASSIGN in 1.9100 sec 2024-11-24T16:41:29,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-24T16:41:29,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9140 sec 2024-11-24T16:41:29,837 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466489837"}]},"ts":"1732466489837"} 2024-11-24T16:41:29,838 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T16:41:29,838 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T16:41:29,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9230 sec 2024-11-24T16:41:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T16:41:30,064 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:30,064 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T16:41:30,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:30,066 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:30,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T16:41:30,066 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:30,069 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,071 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/recovered.edits] 2024-11-24T16:41:30,073 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4695201240784a7ab34cbb9e973b855d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/4695201240784a7ab34cbb9e973b855d 2024-11-24T16:41:30,074 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/631ce7b389a4420cb1218e13048e85b6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/631ce7b389a4420cb1218e13048e85b6 2024-11-24T16:41:30,075 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/9c06ecaac46f4d6188801ac45dc36e9a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/A/9c06ecaac46f4d6188801ac45dc36e9a 2024-11-24T16:41:30,076 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/22bdd6a41aff4ea4b64068f61d1e9741 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/22bdd6a41aff4ea4b64068f61d1e9741 2024-11-24T16:41:30,078 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/86d6428b49e748e5a8192e1dca5ad224 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/86d6428b49e748e5a8192e1dca5ad224 2024-11-24T16:41:30,078 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/ff00e5e87126416ba6ee26e6c4cc6300 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/B/ff00e5e87126416ba6ee26e6c4cc6300 2024-11-24T16:41:30,080 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0fd024efdc96406f8cb2b1fb15b2609f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/0fd024efdc96406f8cb2b1fb15b2609f 2024-11-24T16:41:30,081 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1ffb3f160d4e4162be139c9b53346260 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/1ffb3f160d4e4162be139c9b53346260 2024-11-24T16:41:30,082 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/4402f177d59346a2bdde3474b5860c01 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/C/4402f177d59346a2bdde3474b5860c01 2024-11-24T16:41:30,085 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/recovered.edits/485.seqid to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc/recovered.edits/485.seqid 2024-11-24T16:41:30,085 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T16:41:30,086 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T16:41:30,086 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T16:41:30,090 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241632c654f6fc48de8b9f45de21427cfe_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241632c654f6fc48de8b9f45de21427cfe_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,091 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241a656e5637b44a4ba698c9af811e2223_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241a656e5637b44a4ba698c9af811e2223_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,092 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241bbe6f5701404181aadb31aef2e763e3_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241bbe6f5701404181aadb31aef2e763e3_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,093 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241bff8d9394ab4b25806b96d3abe51c34_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241bff8d9394ab4b25806b96d3abe51c34_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,094 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241dafb4eafaa34e52ab45c5e15835d1ff_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241dafb4eafaa34e52ab45c5e15835d1ff_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,095 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241dc37f9cb1814128ad00de278afd0ee4_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241dc37f9cb1814128ad00de278afd0ee4_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,098 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243e30e2db7e494f199b8fdccb188f9f73_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243e30e2db7e494f199b8fdccb188f9f73_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,099 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b48b268ae464963b05f62654f6c619d_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b48b268ae464963b05f62654f6c619d_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,100 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112461207a03f42148b393b1b3ce55a06ce1_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112461207a03f42148b393b1b3ce55a06ce1_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,101 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247364f7f242ed4d1f999647f2d3d295c1_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247364f7f242ed4d1f999647f2d3d295c1_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,102 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477cfe2b07e75438295f9551414c25b3d_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477cfe2b07e75438295f9551414c25b3d_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,103 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112482b48ab690484fe18edae4be10cc9998_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112482b48ab690484fe18edae4be10cc9998_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,105 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124abaf7eb0cdb84e029af9052554966d04_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124abaf7eb0cdb84e029af9052554966d04_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,106 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b270b6cd1e924cc5b63616ef4b1ca613_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b270b6cd1e924cc5b63616ef4b1ca613_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,107 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c3e53c14bfe446fd9e3dd29ea8bac85e_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c3e53c14bfe446fd9e3dd29ea8bac85e_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,108 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5bbbcbc55be4ee18b1d47a67d223d85_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c5bbbcbc55be4ee18b1d47a67d223d85_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,109 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d94a99719cb14d2cb4d82233d1447ed4_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d94a99719cb14d2cb4d82233d1447ed4_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,110 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dcea88f3d51442f99afe44de92ba134a_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124dcea88f3d51442f99afe44de92ba134a_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,111 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ed9ffebdf9ed449e815cd993dd578d36_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ed9ffebdf9ed449e815cd993dd578d36_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,112 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fc38f6f9119d42fe9f661a2fd58dd15a_68db56cd5236f933146f262f370fc8cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fc38f6f9119d42fe9f661a2fd58dd15a_68db56cd5236f933146f262f370fc8cc 2024-11-24T16:41:30,113 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T16:41:30,115 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:30,118 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T16:41:30,119 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T16:41:30,120 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:30,120 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T16:41:30,121 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732466490120"}]},"ts":"9223372036854775807"} 2024-11-24T16:41:30,123 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T16:41:30,123 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 68db56cd5236f933146f262f370fc8cc, NAME => 'TestAcidGuarantees,,1732466462442.68db56cd5236f933146f262f370fc8cc.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T16:41:30,123 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T16:41:30,123 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732466490123"}]},"ts":"9223372036854775807"} 2024-11-24T16:41:30,124 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T16:41:30,125 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:30,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-24T16:41:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T16:41:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T16:41:30,174 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:30,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249 (was 245) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1194353980_22 at /127.0.0.1:55058 [Waiting for operation #275] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2114816948_22 at /127.0.0.1:36142 [Waiting for operation #305] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2114816948_22 at /127.0.0.1:58432 [Waiting for operation #169] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=503 (was 438) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6736 (was 6516) - AvailableMemoryMB LEAK? - 2024-11-24T16:41:30,191 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=503, ProcessCount=11, AvailableMemoryMB=6736 2024-11-24T16:41:30,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:41:30,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:41:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:30,194 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:41:30,194 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:30,195 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-24T16:41:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T16:41:30,195 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:41:30,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742068_1244 (size=1061) 2024-11-24T16:41:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T16:41:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T16:41:30,603 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:41:30,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742069_1245 (size=53) 2024-11-24T16:41:30,790 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T16:41:30,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T16:41:31,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:41:31,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 200e097340e1bd260123b073b009f0f9, disabling compactions & flushes 2024-11-24T16:41:31,009 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. after waiting 0 ms 2024-11-24T16:41:31,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,009 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,009 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 200e097340e1bd260123b073b009f0f9: Waiting for close lock at 1732466491009Disabling compacts and flushes for region at 1732466491009Disabling writes for close at 1732466491009Writing region close event to WAL at 1732466491009Closed at 1732466491009 2024-11-24T16:41:31,010 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:41:31,010 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732466491010"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466491010"}]},"ts":"1732466491010"} 2024-11-24T16:41:31,011 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:41:31,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:41:31,012 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466491012"}]},"ts":"1732466491012"} 2024-11-24T16:41:31,013 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T16:41:31,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, ASSIGN}] 2024-11-24T16:41:31,014 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, ASSIGN 2024-11-24T16:41:31,015 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, ASSIGN; state=OFFLINE, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=false 2024-11-24T16:41:31,166 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=200e097340e1bd260123b073b009f0f9, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:31,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, ASSIGN because future has completed 2024-11-24T16:41:31,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:31,322 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,322 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:41:31,322 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,322 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:41:31,322 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,322 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,323 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,324 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:31,325 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 200e097340e1bd260123b073b009f0f9 columnFamilyName A 2024-11-24T16:41:31,325 DEBUG [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:31,325 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.HStore(327): Store=200e097340e1bd260123b073b009f0f9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:31,325 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,326 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:31,326 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 200e097340e1bd260123b073b009f0f9 columnFamilyName B 2024-11-24T16:41:31,326 DEBUG [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:31,327 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.HStore(327): Store=200e097340e1bd260123b073b009f0f9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:31,327 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,327 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:31,327 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 200e097340e1bd260123b073b009f0f9 columnFamilyName C 2024-11-24T16:41:31,328 DEBUG [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:31,328 INFO [StoreOpener-200e097340e1bd260123b073b009f0f9-1 {}] regionserver.HStore(327): Store=200e097340e1bd260123b073b009f0f9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:31,328 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,328 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,329 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,329 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,330 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,330 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,330 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:41:31,331 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,332 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:41:31,333 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 200e097340e1bd260123b073b009f0f9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63860866, jitterRate=-0.0483989417552948}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:41:31,333 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:31,334 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 200e097340e1bd260123b073b009f0f9: Running coprocessor pre-open hook at 1732466491322Writing region info on filesystem at 1732466491322Initializing all the Stores at 1732466491323 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466491323Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466491323Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466491323Cleaning up temporary data from old regions at 1732466491330 (+7 ms)Running coprocessor post-open hooks at 1732466491333 (+3 ms)Region opened successfully at 1732466491333 2024-11-24T16:41:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T16:41:31,334 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., pid=63, masterSystemTime=1732466491319 2024-11-24T16:41:31,336 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,336 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:31,336 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=200e097340e1bd260123b073b009f0f9, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:31,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:31,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T16:41:31,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 in 171 msec 2024-11-24T16:41:31,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-24T16:41:31,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, ASSIGN in 325 msec 2024-11-24T16:41:31,341 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:41:31,341 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466491341"}]},"ts":"1732466491341"} 2024-11-24T16:41:31,342 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T16:41:31,343 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:41:31,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-24T16:41:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-24T16:41:32,344 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:32,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12616921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dba3d34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ec22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,348 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,351 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:41:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,37979,1732466430336 2024-11-24T16:41:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46f5e00f 2024-11-24T16:41:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:41:32,352 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:41:32,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c1be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,354 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db05192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,355 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f4692fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27071377, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,358 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344f58d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7da7a03b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78bb7bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,362 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e2e2f23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,363 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b407acb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,365 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcfebd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,366 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b474c80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,368 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30d0cdfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,369 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24eec107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,371 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad78b41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,373 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3909efb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,374 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755924be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,376 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4389e259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,377 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5725862a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,379 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@709bda5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:32,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:32,380 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:32,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54febe41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:32,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:32,381 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:32,382 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf5ae82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:32,382 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,382 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,401 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,401 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,401 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,401 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,403 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,403 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,403 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,404 DEBUG [Thread-1146 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,404 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:32,404 DEBUG [Thread-1146 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,405 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:32,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-24T16:41:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:32,407 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T16:41:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,409 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:32,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:32,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,409 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:41:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:32,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:41:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466552439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466552440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/61f94a362ff442af8ca669a09994e7b0 is 50, key is test_row_0/A:col10/1732466492403/Put/seqid=0 2024-11-24T16:41:32,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466492439, max timestamp=1732466492442Num uniques 859; 2024-11-24T16:41:32,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:32,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466492439, max timestamp=1732466492442Num uniques 859; 2024-11-24T16:41:32,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:32,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466492439, max timestamp=1732466492442Num uniques 859; 2024-11-24T16:41:32,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742070_1246 (size=16681) 2024-11-24T16:41:32,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/61f94a362ff442af8ca669a09994e7b0 2024-11-24T16:41:32,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466552483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466552484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466552484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T16:41:32,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/f357cc5fff2d4bc4ad7cd0cf208ea313 is 50, key is test_row_0/B:col10/1732466492403/Put/seqid=0 2024-11-24T16:41:32,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742071_1247 (size=12001) 2024-11-24T16:41:32,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/f357cc5fff2d4bc4ad7cd0cf208ea313 2024-11-24T16:41:32,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466552554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466552554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T16:41:32,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:32,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:32,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:32,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/563ba19fdb76434f97025cbb5cc296ed is 50, key is test_row_0/C:col10/1732466492403/Put/seqid=0 2024-11-24T16:41:32,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742072_1248 (size=12001) 2024-11-24T16:41:32,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466552594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466552604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466552604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T16:41:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:32,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T16:41:32,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466552763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466552763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466552804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466552813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466552813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:32,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T16:41:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:32,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:32,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/563ba19fdb76434f97025cbb5cc296ed 2024-11-24T16:41:32,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/61f94a362ff442af8ca669a09994e7b0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/61f94a362ff442af8ca669a09994e7b0 2024-11-24T16:41:32,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/61f94a362ff442af8ca669a09994e7b0, entries=250, sequenceid=13, filesize=16.3 K 2024-11-24T16:41:32,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/f357cc5fff2d4bc4ad7cd0cf208ea313 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f357cc5fff2d4bc4ad7cd0cf208ea313 2024-11-24T16:41:32,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f357cc5fff2d4bc4ad7cd0cf208ea313, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T16:41:32,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/563ba19fdb76434f97025cbb5cc296ed as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/563ba19fdb76434f97025cbb5cc296ed 2024-11-24T16:41:33,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/563ba19fdb76434f97025cbb5cc296ed, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T16:41:33,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 200e097340e1bd260123b073b009f0f9 in 585ms, sequenceid=13, compaction requested=false 2024-11-24T16:41:33,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:33,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T16:41:33,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:33,025 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T16:41:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:33,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/64366a0f071a42e09d73765403a79f8d is 50, key is test_row_0/A:col10/1732466492481/Put/seqid=0 2024-11-24T16:41:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T16:41:33,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742073_1249 (size=12001) 2024-11-24T16:41:33,045 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/64366a0f071a42e09d73765403a79f8d 2024-11-24T16:41:33,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/73412c5b86654edb8858f04e5f6efbc2 is 50, key is test_row_0/B:col10/1732466492481/Put/seqid=0 2024-11-24T16:41:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:33,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742074_1250 (size=12001) 2024-11-24T16:41:33,080 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/73412c5b86654edb8858f04e5f6efbc2 2024-11-24T16:41:33,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/9dc189d8881b4f3db072fec6470cb3db is 50, key is test_row_0/C:col10/1732466492481/Put/seqid=0 2024-11-24T16:41:33,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742075_1251 (size=12001) 2024-11-24T16:41:33,102 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/9dc189d8881b4f3db072fec6470cb3db 2024-11-24T16:41:33,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466553100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466553101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/64366a0f071a42e09d73765403a79f8d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/64366a0f071a42e09d73765403a79f8d 2024-11-24T16:41:33,116 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/64366a0f071a42e09d73765403a79f8d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T16:41:33,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/73412c5b86654edb8858f04e5f6efbc2 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/73412c5b86654edb8858f04e5f6efbc2 2024-11-24T16:41:33,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466553113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,122 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/73412c5b86654edb8858f04e5f6efbc2, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T16:41:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/9dc189d8881b4f3db072fec6470cb3db as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9dc189d8881b4f3db072fec6470cb3db 2024-11-24T16:41:33,127 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9dc189d8881b4f3db072fec6470cb3db, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T16:41:33,128 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 200e097340e1bd260123b073b009f0f9 in 103ms, sequenceid=49, compaction requested=false 2024-11-24T16:41:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-24T16:41:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-24T16:41:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:33,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T16:41:33,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:33,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:33,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:33,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:33,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:33,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:33,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-24T16:41:33,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 722 msec 2024-11-24T16:41:33,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 730 msec 2024-11-24T16:41:33,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/7233780298304ba8ab8d167b8710e8ac is 50, key is test_row_0/A:col10/1732466493130/Put/seqid=0 2024-11-24T16:41:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742076_1252 (size=12001) 2024-11-24T16:41:33,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466553196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466553203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466553213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466553214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466553313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466553313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466553423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466553423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466553523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466553524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T16:41:33,544 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:33,545 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:33,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-24T16:41:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T16:41:33,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:33,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:33,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:33,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/7233780298304ba8ab8d167b8710e8ac 2024-11-24T16:41:33,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/1e8539507bfa4fa68155ed1af80d854c is 50, key is test_row_0/B:col10/1732466493130/Put/seqid=0 2024-11-24T16:41:33,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742077_1253 (size=12001) 2024-11-24T16:41:33,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466553634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T16:41:33,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T16:41:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:33,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:33,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:33,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:33,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466553734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466553734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466553834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466553834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:33,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T16:41:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:33,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:33,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T16:41:33,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/1e8539507bfa4fa68155ed1af80d854c 2024-11-24T16:41:33,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/f913158ed0d946018af1ffe85a3d8075 is 50, key is test_row_0/C:col10/1732466493130/Put/seqid=0 2024-11-24T16:41:33,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742078_1254 (size=12001) 2024-11-24T16:41:33,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/f913158ed0d946018af1ffe85a3d8075 2024-11-24T16:41:33,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/7233780298304ba8ab8d167b8710e8ac as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/7233780298304ba8ab8d167b8710e8ac 2024-11-24T16:41:33,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/7233780298304ba8ab8d167b8710e8ac, entries=150, sequenceid=62, filesize=11.7 K 2024-11-24T16:41:33,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/1e8539507bfa4fa68155ed1af80d854c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1e8539507bfa4fa68155ed1af80d854c 2024-11-24T16:41:33,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1e8539507bfa4fa68155ed1af80d854c, entries=150, sequenceid=62, filesize=11.7 K 2024-11-24T16:41:33,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/f913158ed0d946018af1ffe85a3d8075 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f913158ed0d946018af1ffe85a3d8075 2024-11-24T16:41:34,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f913158ed0d946018af1ffe85a3d8075, entries=150, sequenceid=62, filesize=11.7 K 2024-11-24T16:41:34,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 200e097340e1bd260123b073b009f0f9 in 874ms, sequenceid=62, compaction requested=true 2024-11-24T16:41:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:34,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T16:41:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:34,005 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:34,006 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:34,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,006 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T16:41:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:34,007 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:34,007 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:34,007 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,008 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f357cc5fff2d4bc4ad7cd0cf208ea313, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/73412c5b86654edb8858f04e5f6efbc2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1e8539507bfa4fa68155ed1af80d854c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=35.2 K 2024-11-24T16:41:34,008 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:34,008 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:34,008 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,008 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/61f94a362ff442af8ca669a09994e7b0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/64366a0f071a42e09d73765403a79f8d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/7233780298304ba8ab8d167b8710e8ac] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=39.7 K 2024-11-24T16:41:34,009 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f357cc5fff2d4bc4ad7cd0cf208ea313, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466492403 2024-11-24T16:41:34,010 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61f94a362ff442af8ca669a09994e7b0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466492403 2024-11-24T16:41:34,010 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 73412c5b86654edb8858f04e5f6efbc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732466492473 2024-11-24T16:41:34,010 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64366a0f071a42e09d73765403a79f8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732466492473 2024-11-24T16:41:34,010 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e8539507bfa4fa68155ed1af80d854c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732466493099 2024-11-24T16:41:34,011 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7233780298304ba8ab8d167b8710e8ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732466493099 2024-11-24T16:41:34,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/cb418da03a574f4a942668aa0832494c is 50, key is test_row_0/A:col10/1732466493145/Put/seqid=0 2024-11-24T16:41:34,034 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:34,035 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/1c88afbdf72a4709b66b9f5bca61bdc0 is 50, key is test_row_0/A:col10/1732466493130/Put/seqid=0 2024-11-24T16:41:34,039 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:34,039 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/cfa12b5f4d3c4a4fb631650626a00ca1 is 50, key is test_row_0/B:col10/1732466493130/Put/seqid=0 2024-11-24T16:41:34,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742079_1255 (size=12001) 2024-11-24T16:41:34,073 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/cb418da03a574f4a942668aa0832494c 2024-11-24T16:41:34,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742080_1256 (size=12104) 2024-11-24T16:41:34,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742081_1257 (size=12104) 2024-11-24T16:41:34,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/4c542dc9adaf43a284168bf63f4d7593 is 50, key is test_row_0/B:col10/1732466493145/Put/seqid=0 2024-11-24T16:41:34,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742082_1258 (size=12001) 2024-11-24T16:41:34,116 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/4c542dc9adaf43a284168bf63f4d7593 2024-11-24T16:41:34,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/99cc27e4cc834dffb4205ba5d3089879 is 50, key is test_row_0/C:col10/1732466493145/Put/seqid=0 2024-11-24T16:41:34,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742083_1259 (size=12001) 2024-11-24T16:41:34,144 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/99cc27e4cc834dffb4205ba5d3089879 2024-11-24T16:41:34,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/cb418da03a574f4a942668aa0832494c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/cb418da03a574f4a942668aa0832494c 2024-11-24T16:41:34,159 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/cb418da03a574f4a942668aa0832494c, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T16:41:34,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/4c542dc9adaf43a284168bf63f4d7593 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4c542dc9adaf43a284168bf63f4d7593 2024-11-24T16:41:34,169 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4c542dc9adaf43a284168bf63f4d7593, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T16:41:34,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/99cc27e4cc834dffb4205ba5d3089879 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/99cc27e4cc834dffb4205ba5d3089879 2024-11-24T16:41:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T16:41:34,182 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/99cc27e4cc834dffb4205ba5d3089879, entries=150, sequenceid=81, filesize=11.7 K 2024-11-24T16:41:34,184 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 200e097340e1bd260123b073b009f0f9 in 178ms, sequenceid=81, compaction requested=true 2024-11-24T16:41:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-24T16:41:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-24T16:41:34,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-24T16:41:34,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 639 msec 2024-11-24T16:41:34,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 645 msec 2024-11-24T16:41:34,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:34,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:41:34,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:34,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:34,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:34,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:34,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:34,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:34,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/5018dcd3cde446eb90c3dcf07f808f98 is 50, key is test_row_0/A:col10/1732466494266/Put/seqid=0 2024-11-24T16:41:34,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742084_1260 (size=14341) 2024-11-24T16:41:34,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/5018dcd3cde446eb90c3dcf07f808f98 2024-11-24T16:41:34,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac is 50, key is test_row_0/B:col10/1732466494266/Put/seqid=0 2024-11-24T16:41:34,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742085_1261 (size=12001) 2024-11-24T16:41:34,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac 2024-11-24T16:41:34,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466494305, max timestamp=1732466494330Num uniques 859; 2024-11-24T16:41:34,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466494305, max timestamp=1732466494330Num uniques 859; 2024-11-24T16:41:34,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:34,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466494305, max timestamp=1732466494330Num uniques 859; 2024-11-24T16:41:34,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:34,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/f0df926ccf8e4bdf8419123eeda10f46 is 50, key is test_row_0/C:col10/1732466494266/Put/seqid=0 2024-11-24T16:41:34,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742086_1262 (size=12001) 2024-11-24T16:41:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466554369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466554371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466554372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466554373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466554484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466554484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466554484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466554484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,496 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/1c88afbdf72a4709b66b9f5bca61bdc0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1c88afbdf72a4709b66b9f5bca61bdc0 2024-11-24T16:41:34,499 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/cfa12b5f4d3c4a4fb631650626a00ca1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/cfa12b5f4d3c4a4fb631650626a00ca1 2024-11-24T16:41:34,504 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 1c88afbdf72a4709b66b9f5bca61bdc0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:34,504 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:34,504 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466494005; duration=0sec 2024-11-24T16:41:34,504 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:34,504 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:34,504 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:34,505 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into cfa12b5f4d3c4a4fb631650626a00ca1(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:34,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:34,505 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:34,505 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466494005; duration=0sec 2024-11-24T16:41:34,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:34,505 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:34,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:34,506 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,506 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/563ba19fdb76434f97025cbb5cc296ed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9dc189d8881b4f3db072fec6470cb3db, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f913158ed0d946018af1ffe85a3d8075, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/99cc27e4cc834dffb4205ba5d3089879] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=46.9 K 2024-11-24T16:41:34,506 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563ba19fdb76434f97025cbb5cc296ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466492403 2024-11-24T16:41:34,506 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dc189d8881b4f3db072fec6470cb3db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732466492473 2024-11-24T16:41:34,508 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting f913158ed0d946018af1ffe85a3d8075, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732466493099 2024-11-24T16:41:34,509 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99cc27e4cc834dffb4205ba5d3089879, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732466493145 2024-11-24T16:41:34,522 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:34,523 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/586d0973cbe84d96a3d9116dee1e54c3 is 50, key is test_row_0/C:col10/1732466493145/Put/seqid=0 2024-11-24T16:41:34,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742087_1263 (size=12139) 2024-11-24T16:41:34,542 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/586d0973cbe84d96a3d9116dee1e54c3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/586d0973cbe84d96a3d9116dee1e54c3 2024-11-24T16:41:34,547 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into 586d0973cbe84d96a3d9116dee1e54c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:34,547 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:34,547 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=12, startTime=1732466494006; duration=0sec 2024-11-24T16:41:34,547 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:34,547 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:34,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466554644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,648 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T16:41:34,684 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:34,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-24T16:41:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T16:41:34,686 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:34,687 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:34,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466554694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466554694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466554694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466554703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:34,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:34,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/f0df926ccf8e4bdf8419123eeda10f46 2024-11-24T16:41:34,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/5018dcd3cde446eb90c3dcf07f808f98 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/5018dcd3cde446eb90c3dcf07f808f98 2024-11-24T16:41:34,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/5018dcd3cde446eb90c3dcf07f808f98, entries=200, sequenceid=91, filesize=14.0 K 2024-11-24T16:41:34,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac 2024-11-24T16:41:34,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac, entries=150, sequenceid=91, filesize=11.7 K 2024-11-24T16:41:34,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/f0df926ccf8e4bdf8419123eeda10f46 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f0df926ccf8e4bdf8419123eeda10f46 2024-11-24T16:41:34,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f0df926ccf8e4bdf8419123eeda10f46, entries=150, sequenceid=91, filesize=11.7 K 2024-11-24T16:41:34,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 200e097340e1bd260123b073b009f0f9 in 519ms, sequenceid=91, compaction requested=true 2024-11-24T16:41:34,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:34,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:34,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:34,787 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:34,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:34,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:34,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:34,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T16:41:34,787 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:34,788 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:34,788 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:34,788 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:34,788 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:34,788 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,788 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,788 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/cfa12b5f4d3c4a4fb631650626a00ca1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4c542dc9adaf43a284168bf63f4d7593, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=35.3 K 2024-11-24T16:41:34,788 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1c88afbdf72a4709b66b9f5bca61bdc0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/cb418da03a574f4a942668aa0832494c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/5018dcd3cde446eb90c3dcf07f808f98] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=37.5 K 2024-11-24T16:41:34,791 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting cfa12b5f4d3c4a4fb631650626a00ca1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732466493099 2024-11-24T16:41:34,791 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c88afbdf72a4709b66b9f5bca61bdc0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732466493099 2024-11-24T16:41:34,792 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb418da03a574f4a942668aa0832494c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732466493145 2024-11-24T16:41:34,792 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c542dc9adaf43a284168bf63f4d7593, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732466493145 2024-11-24T16:41:34,792 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting b76e6ea1931d44c7bd7ab43ab3f9c2ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732466494258 2024-11-24T16:41:34,792 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5018dcd3cde446eb90c3dcf07f808f98, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732466494258 2024-11-24T16:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T16:41:34,800 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:34,800 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:34,800 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/2e3bd9b2546446449378ea00a9601cbf is 50, key is test_row_0/B:col10/1732466494266/Put/seqid=0 2024-11-24T16:41:34,801 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/8631f8bd5e484ad48bc631ac495f4e52 is 50, key is test_row_0/A:col10/1732466494266/Put/seqid=0 2024-11-24T16:41:34,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742088_1264 (size=12207) 2024-11-24T16:41:34,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742089_1265 (size=12207) 2024-11-24T16:41:34,818 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/8631f8bd5e484ad48bc631ac495f4e52 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/8631f8bd5e484ad48bc631ac495f4e52 2024-11-24T16:41:34,823 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 8631f8bd5e484ad48bc631ac495f4e52(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:34,823 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:34,823 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466494787; duration=0sec 2024-11-24T16:41:34,823 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:34,823 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:34,824 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T16:41:34,824 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T16:41:34,824 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T16:41:34,825 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. because compaction request was cancelled 2024-11-24T16:41:34,825 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:34,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T16:41:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:34,839 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T16:41:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:34,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:34,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/abfab1dd8e4d400fa58879a09a75ec5e is 50, key is test_row_0/A:col10/1732466494372/Put/seqid=0 2024-11-24T16:41:34,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742090_1266 (size=12001) 2024-11-24T16:41:34,852 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/abfab1dd8e4d400fa58879a09a75ec5e 2024-11-24T16:41:34,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/5d77211a3b634000bfe4c335df711030 is 50, key is test_row_0/B:col10/1732466494372/Put/seqid=0 2024-11-24T16:41:34,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742091_1267 (size=12001) 2024-11-24T16:41:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T16:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:35,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466555014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466555014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466555016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466555019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466555124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466555124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466555133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,213 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/2e3bd9b2546446449378ea00a9601cbf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/2e3bd9b2546446449378ea00a9601cbf 2024-11-24T16:41:35,221 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into 2e3bd9b2546446449378ea00a9601cbf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:35,221 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:35,221 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466494787; duration=0sec 2024-11-24T16:41:35,221 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:35,221 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:35,287 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/5d77211a3b634000bfe4c335df711030 2024-11-24T16:41:35,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/7e57d00ea02447d3a1c17dca0dd6b936 is 50, key is test_row_0/C:col10/1732466494372/Put/seqid=0 2024-11-24T16:41:35,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742092_1268 (size=12001) 2024-11-24T16:41:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T16:41:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466555334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466555335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466555343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,348 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:41:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466555533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466555644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466555644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466555653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:35,702 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/7e57d00ea02447d3a1c17dca0dd6b936 2024-11-24T16:41:35,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/abfab1dd8e4d400fa58879a09a75ec5e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/abfab1dd8e4d400fa58879a09a75ec5e 2024-11-24T16:41:35,716 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/abfab1dd8e4d400fa58879a09a75ec5e, entries=150, sequenceid=132, filesize=11.7 K 2024-11-24T16:41:35,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/5d77211a3b634000bfe4c335df711030 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/5d77211a3b634000bfe4c335df711030 2024-11-24T16:41:35,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/5d77211a3b634000bfe4c335df711030, entries=150, sequenceid=132, filesize=11.7 K 2024-11-24T16:41:35,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/7e57d00ea02447d3a1c17dca0dd6b936 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7e57d00ea02447d3a1c17dca0dd6b936 2024-11-24T16:41:35,749 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7e57d00ea02447d3a1c17dca0dd6b936, entries=150, sequenceid=132, filesize=11.7 K 2024-11-24T16:41:35,750 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 200e097340e1bd260123b073b009f0f9 in 911ms, sequenceid=132, compaction requested=true 2024-11-24T16:41:35,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:35,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:35,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-24T16:41:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-24T16:41:35,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T16:41:35,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0670 sec 2024-11-24T16:41:35,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0740 sec 2024-11-24T16:41:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T16:41:35,825 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:35,826 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-24T16:41:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T16:41:35,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:35,833 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:35,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T16:41:35,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T16:41:35,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:35,987 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:41:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:35,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/74bd58d6b5e041aaa2065bbdd2af02a7 is 50, key is test_row_0/A:col10/1732466495017/Put/seqid=0 2024-11-24T16:41:36,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742093_1269 (size=12151) 2024-11-24T16:41:36,020 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/74bd58d6b5e041aaa2065bbdd2af02a7 2024-11-24T16:41:36,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/63a08e1fc10b4b54be2f6fd185fec924 is 50, key is test_row_0/B:col10/1732466495017/Put/seqid=0 2024-11-24T16:41:36,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742094_1270 (size=12151) 2024-11-24T16:41:36,051 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/63a08e1fc10b4b54be2f6fd185fec924 2024-11-24T16:41:36,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b55e11d6aed14bedbcd4938a2116b2b3 is 50, key is test_row_0/C:col10/1732466495017/Put/seqid=0 2024-11-24T16:41:36,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742095_1271 (size=12151) 2024-11-24T16:41:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T16:41:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:36,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:36,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466496210, max timestamp=1732466496227Num uniques 859; 2024-11-24T16:41:36,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:36,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466496210, max timestamp=1732466496227Num uniques 859; 2024-11-24T16:41:36,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:36,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732466496210, max timestamp=1732466496227Num uniques 859; 2024-11-24T16:41:36,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-24T16:41:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466556267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466556268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466556270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466556373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466556384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466556385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T16:41:36,507 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b55e11d6aed14bedbcd4938a2116b2b3 2024-11-24T16:41:36,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/74bd58d6b5e041aaa2065bbdd2af02a7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/74bd58d6b5e041aaa2065bbdd2af02a7 2024-11-24T16:41:36,520 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/74bd58d6b5e041aaa2065bbdd2af02a7, entries=150, sequenceid=143, filesize=11.9 K 2024-11-24T16:41:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/63a08e1fc10b4b54be2f6fd185fec924 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/63a08e1fc10b4b54be2f6fd185fec924 2024-11-24T16:41:36,528 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/63a08e1fc10b4b54be2f6fd185fec924, entries=150, sequenceid=143, filesize=11.9 K 2024-11-24T16:41:36,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b55e11d6aed14bedbcd4938a2116b2b3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b55e11d6aed14bedbcd4938a2116b2b3 2024-11-24T16:41:36,534 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b55e11d6aed14bedbcd4938a2116b2b3, entries=150, sequenceid=143, filesize=11.9 K 2024-11-24T16:41:36,535 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 200e097340e1bd260123b073b009f0f9 in 549ms, sequenceid=143, compaction requested=true 2024-11-24T16:41:36,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:36,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:36,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-24T16:41:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-24T16:41:36,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T16:41:36,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 705 msec 2024-11-24T16:41:36,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 716 msec 2024-11-24T16:41:36,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466496259, max timestamp=1732466496547Num uniques 1018; 2024-11-24T16:41:36,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466496259, max timestamp=1732466496547Num uniques 1018; 2024-11-24T16:41:36,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:41:36,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:41:36,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466496259, max timestamp=1732466496547Num uniques 1009; 2024-11-24T16:41:36,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-24T16:41:36,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-11-24T16:41:36,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:36,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:36,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:36,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:36,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:36,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:36,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/65e71da7408f4a9c94dcdacace481d2b is 50, key is test_row_0/A:col10/1732466496570/Put/seqid=0 2024-11-24T16:41:36,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742096_1272 (size=12151) 2024-11-24T16:41:36,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466496625, max timestamp=1732466496639Num uniques 859; 2024-11-24T16:41:36,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466496625, max timestamp=1732466496639Num uniques 859; 2024-11-24T16:41:36,643 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:36,644 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:36,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466496625, max timestamp=1732466496639Num uniques 859; 2024-11-24T16:41:36,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:36,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466556681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466556682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466556683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466556693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466556694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466556804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466556804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466556804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466556805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466556814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T16:41:36,964 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:36,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-24T16:41:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T16:41:36,966 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:36,967 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:36,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466557014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466557014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466557014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466557014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466557023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/65e71da7408f4a9c94dcdacace481d2b 2024-11-24T16:41:37,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e4ceda7db84b4ee58a79332d1c0082f6 is 50, key is test_row_0/B:col10/1732466496570/Put/seqid=0 2024-11-24T16:41:37,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742097_1273 (size=12151) 2024-11-24T16:41:37,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e4ceda7db84b4ee58a79332d1c0082f6 2024-11-24T16:41:37,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/d9fd466ee83a4d55ad1c30d0b73d8bbe is 50, key is test_row_0/C:col10/1732466496570/Put/seqid=0 2024-11-24T16:41:37,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742098_1274 (size=12151) 2024-11-24T16:41:37,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T16:41:37,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T16:41:37,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:37,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T16:41:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T16:41:37,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466557323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466557334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466557334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466557334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466557334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T16:41:37,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:37,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:37,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/d9fd466ee83a4d55ad1c30d0b73d8bbe 2024-11-24T16:41:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/65e71da7408f4a9c94dcdacace481d2b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/65e71da7408f4a9c94dcdacace481d2b 2024-11-24T16:41:37,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/65e71da7408f4a9c94dcdacace481d2b, entries=150, sequenceid=185, filesize=11.9 K 2024-11-24T16:41:37,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e4ceda7db84b4ee58a79332d1c0082f6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e4ceda7db84b4ee58a79332d1c0082f6 2024-11-24T16:41:37,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e4ceda7db84b4ee58a79332d1c0082f6, entries=150, sequenceid=185, filesize=11.9 K 2024-11-24T16:41:37,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/d9fd466ee83a4d55ad1c30d0b73d8bbe as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d9fd466ee83a4d55ad1c30d0b73d8bbe 2024-11-24T16:41:37,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d9fd466ee83a4d55ad1c30d0b73d8bbe, entries=150, sequenceid=185, filesize=11.9 K 2024-11-24T16:41:37,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.03 KB/148512, currentSize=134.18 KB/137400 for 200e097340e1bd260123b073b009f0f9 in 917ms, sequenceid=185, compaction requested=true 2024-11-24T16:41:37,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:37,491 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:37,491 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:37,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:37,492 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:37,492 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:37,492 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,492 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/2e3bd9b2546446449378ea00a9601cbf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/5d77211a3b634000bfe4c335df711030, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/63a08e1fc10b4b54be2f6fd185fec924, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e4ceda7db84b4ee58a79332d1c0082f6] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=47.4 K 2024-11-24T16:41:37,493 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:37,493 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:37,493 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,493 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/8631f8bd5e484ad48bc631ac495f4e52, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/abfab1dd8e4d400fa58879a09a75ec5e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/74bd58d6b5e041aaa2065bbdd2af02a7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/65e71da7408f4a9c94dcdacace481d2b] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=47.4 K 2024-11-24T16:41:37,493 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8631f8bd5e484ad48bc631ac495f4e52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732466494258 2024-11-24T16:41:37,494 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3bd9b2546446449378ea00a9601cbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732466494258 2024-11-24T16:41:37,494 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting abfab1dd8e4d400fa58879a09a75ec5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732466494359 2024-11-24T16:41:37,495 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d77211a3b634000bfe4c335df711030, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732466494359 2024-11-24T16:41:37,495 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74bd58d6b5e041aaa2065bbdd2af02a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732466495013 2024-11-24T16:41:37,495 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a08e1fc10b4b54be2f6fd185fec924, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732466495013 2024-11-24T16:41:37,495 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e71da7408f4a9c94dcdacace481d2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732466496265 2024-11-24T16:41:37,496 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e4ceda7db84b4ee58a79332d1c0082f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732466496265 2024-11-24T16:41:37,509 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#230 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:37,510 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/181a65ca9d3d47ca96e9118c1e0399fb is 50, key is test_row_0/B:col10/1732466496570/Put/seqid=0 2024-11-24T16:41:37,513 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#231 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:37,513 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/f5a05e122cf048558e4e93c1a16099c2 is 50, key is test_row_0/A:col10/1732466496570/Put/seqid=0 2024-11-24T16:41:37,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742099_1275 (size=12493) 2024-11-24T16:41:37,526 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/181a65ca9d3d47ca96e9118c1e0399fb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/181a65ca9d3d47ca96e9118c1e0399fb 2024-11-24T16:41:37,531 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into 181a65ca9d3d47ca96e9118c1e0399fb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:37,531 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:37,531 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=12, startTime=1732466497491; duration=0sec 2024-11-24T16:41:37,531 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:37,531 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:37,531 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T16:41:37,534 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60443 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T16:41:37,534 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:37,534 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,534 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/586d0973cbe84d96a3d9116dee1e54c3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f0df926ccf8e4bdf8419123eeda10f46, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7e57d00ea02447d3a1c17dca0dd6b936, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b55e11d6aed14bedbcd4938a2116b2b3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d9fd466ee83a4d55ad1c30d0b73d8bbe] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=59.0 K 2024-11-24T16:41:37,535 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 586d0973cbe84d96a3d9116dee1e54c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732466493145 2024-11-24T16:41:37,535 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f0df926ccf8e4bdf8419123eeda10f46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732466494258 2024-11-24T16:41:37,535 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e57d00ea02447d3a1c17dca0dd6b936, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732466494359 2024-11-24T16:41:37,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742100_1276 (size=12493) 2024-11-24T16:41:37,536 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting b55e11d6aed14bedbcd4938a2116b2b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732466495013 2024-11-24T16:41:37,536 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d9fd466ee83a4d55ad1c30d0b73d8bbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732466496265 2024-11-24T16:41:37,543 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/f5a05e122cf048558e4e93c1a16099c2 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5a05e122cf048558e4e93c1a16099c2 2024-11-24T16:41:37,551 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into f5a05e122cf048558e4e93c1a16099c2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:37,552 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:37,552 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=12, startTime=1732466497491; duration=0sec 2024-11-24T16:41:37,552 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:37,552 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:37,559 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:37,560 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/12d0877db0874ee8a2aab8fdeb3b8e2d is 50, key is test_row_0/C:col10/1732466496570/Put/seqid=0 2024-11-24T16:41:37,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742101_1277 (size=12459) 2024-11-24T16:41:37,573 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/12d0877db0874ee8a2aab8fdeb3b8e2d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/12d0877db0874ee8a2aab8fdeb3b8e2d 2024-11-24T16:41:37,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T16:41:37,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:37,580 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T16:41:37,580 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into 12d0877db0874ee8a2aab8fdeb3b8e2d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:37,580 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:37,581 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=11, startTime=1732466497491; duration=0sec 2024-11-24T16:41:37,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:37,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:37,581 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:37,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:37,581 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:37,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:37,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:37,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:37,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/6bfb8117277740b3bff09881b8bfef47 is 50, key is test_row_0/A:col10/1732466496692/Put/seqid=0 2024-11-24T16:41:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T16:41:37,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742102_1278 (size=12151) 2024-11-24T16:41:37,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/6bfb8117277740b3bff09881b8bfef47 2024-11-24T16:41:37,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/4f8cc86366d84b049d4e6f19c9912d36 is 50, key is test_row_0/B:col10/1732466496692/Put/seqid=0 2024-11-24T16:41:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742103_1279 (size=12151) 2024-11-24T16:41:37,639 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/4f8cc86366d84b049d4e6f19c9912d36 2024-11-24T16:41:37,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/0b38d75c52a744fdb8669855884748e3 is 50, key is test_row_0/C:col10/1732466496692/Put/seqid=0 2024-11-24T16:41:37,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742104_1280 (size=12151) 2024-11-24T16:41:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:37,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466557850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466557851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466557851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466557856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466557857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466557963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466557963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466557964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466557964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466557964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:37,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,074 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/0b38d75c52a744fdb8669855884748e3 2024-11-24T16:41:38,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/6bfb8117277740b3bff09881b8bfef47 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6bfb8117277740b3bff09881b8bfef47 2024-11-24T16:41:38,088 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6bfb8117277740b3bff09881b8bfef47, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T16:41:38,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/4f8cc86366d84b049d4e6f19c9912d36 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4f8cc86366d84b049d4e6f19c9912d36 2024-11-24T16:41:38,097 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4f8cc86366d84b049d4e6f19c9912d36, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T16:41:38,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/0b38d75c52a744fdb8669855884748e3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/0b38d75c52a744fdb8669855884748e3 2024-11-24T16:41:38,103 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/0b38d75c52a744fdb8669855884748e3, entries=150, sequenceid=225, filesize=11.9 K 2024-11-24T16:41:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T16:41:38,105 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 200e097340e1bd260123b073b009f0f9 in 525ms, sequenceid=225, compaction requested=false 2024-11-24T16:41:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-24T16:41:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-24T16:41:38,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T16:41:38,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1390 sec 2024-11-24T16:41:38,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1470 sec 2024-11-24T16:41:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:38,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:41:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:38,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/9260c524142b4c5da3bfb72c123b3225 is 50, key is test_row_0/A:col10/1732466497854/Put/seqid=0 2024-11-24T16:41:38,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742105_1281 (size=12151) 2024-11-24T16:41:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466558204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466558206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466558214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466558217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466558214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466558324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466558324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466558333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466558333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466558334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466558534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466558534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466558544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466558544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466558544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/9260c524142b4c5da3bfb72c123b3225 2024-11-24T16:41:38,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/fa8a5b1bfaf74547ac3a7ce18109a130 is 50, key is test_row_0/B:col10/1732466497854/Put/seqid=0 2024-11-24T16:41:38,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742106_1282 (size=12151) 2024-11-24T16:41:38,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466558844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466558844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466558853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466558853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466558853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/fa8a5b1bfaf74547ac3a7ce18109a130 2024-11-24T16:41:39,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a0def2a311244e4eb702625ea87ee4aa is 50, key is test_row_0/C:col10/1732466497854/Put/seqid=0 2024-11-24T16:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T16:41:39,114 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:39,115 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-24T16:41:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T16:41:39,117 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:39,119 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:39,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:39,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742107_1283 (size=12151) 2024-11-24T16:41:39,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a0def2a311244e4eb702625ea87ee4aa 2024-11-24T16:41:39,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/9260c524142b4c5da3bfb72c123b3225 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/9260c524142b4c5da3bfb72c123b3225 2024-11-24T16:41:39,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/9260c524142b4c5da3bfb72c123b3225, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T16:41:39,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/fa8a5b1bfaf74547ac3a7ce18109a130 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/fa8a5b1bfaf74547ac3a7ce18109a130 2024-11-24T16:41:39,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/fa8a5b1bfaf74547ac3a7ce18109a130, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T16:41:39,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a0def2a311244e4eb702625ea87ee4aa as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a0def2a311244e4eb702625ea87ee4aa 2024-11-24T16:41:39,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a0def2a311244e4eb702625ea87ee4aa, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T16:41:39,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 200e097340e1bd260123b073b009f0f9 in 982ms, sequenceid=237, compaction requested=true 2024-11-24T16:41:39,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:39,160 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:39,160 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:39,161 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:39,161 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:39,161 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,161 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5a05e122cf048558e4e93c1a16099c2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6bfb8117277740b3bff09881b8bfef47, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/9260c524142b4c5da3bfb72c123b3225] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=35.9 K 2024-11-24T16:41:39,161 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:39,161 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:39,161 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,161 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/181a65ca9d3d47ca96e9118c1e0399fb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4f8cc86366d84b049d4e6f19c9912d36, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/fa8a5b1bfaf74547ac3a7ce18109a130] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=35.9 K 2024-11-24T16:41:39,162 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a05e122cf048558e4e93c1a16099c2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732466496265 2024-11-24T16:41:39,162 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 181a65ca9d3d47ca96e9118c1e0399fb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732466496265 2024-11-24T16:41:39,162 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bfb8117277740b3bff09881b8bfef47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732466496679 2024-11-24T16:41:39,162 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8cc86366d84b049d4e6f19c9912d36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732466496679 2024-11-24T16:41:39,163 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fa8a5b1bfaf74547ac3a7ce18109a130, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732466497854 2024-11-24T16:41:39,163 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9260c524142b4c5da3bfb72c123b3225, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732466497854 2024-11-24T16:41:39,183 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:39,183 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/ccc21e441b2e410db4508181110d190d is 50, key is test_row_0/B:col10/1732466497854/Put/seqid=0 2024-11-24T16:41:39,211 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:39,211 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/b14d77dd93fb4cd9ac243740b370316a is 50, key is test_row_0/A:col10/1732466497854/Put/seqid=0 2024-11-24T16:41:39,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742108_1284 (size=12595) 2024-11-24T16:41:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T16:41:39,225 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/ccc21e441b2e410db4508181110d190d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/ccc21e441b2e410db4508181110d190d 2024-11-24T16:41:39,233 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into ccc21e441b2e410db4508181110d190d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:39,233 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:39,233 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466499160; duration=0sec 2024-11-24T16:41:39,233 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:39,233 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:39,233 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:39,234 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:39,234 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:39,234 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,234 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/12d0877db0874ee8a2aab8fdeb3b8e2d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/0b38d75c52a744fdb8669855884748e3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a0def2a311244e4eb702625ea87ee4aa] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=35.9 K 2024-11-24T16:41:39,235 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d0877db0874ee8a2aab8fdeb3b8e2d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732466496265 2024-11-24T16:41:39,236 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b38d75c52a744fdb8669855884748e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732466496679 2024-11-24T16:41:39,236 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a0def2a311244e4eb702625ea87ee4aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732466497854 2024-11-24T16:41:39,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742109_1285 (size=12595) 2024-11-24T16:41:39,249 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/b14d77dd93fb4cd9ac243740b370316a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/b14d77dd93fb4cd9ac243740b370316a 2024-11-24T16:41:39,255 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into b14d77dd93fb4cd9ac243740b370316a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:39,255 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:39,255 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466499160; duration=0sec 2024-11-24T16:41:39,255 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:39,255 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:39,258 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:39,258 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/ccfad34dc5084ad1a23b8348baef16a7 is 50, key is test_row_0/C:col10/1732466497854/Put/seqid=0 2024-11-24T16:41:39,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T16:41:39,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,278 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:41:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:39,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/34ba6f87891945cba45f25139a20978f is 50, key is test_row_0/A:col10/1732466498211/Put/seqid=0 2024-11-24T16:41:39,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742110_1286 (size=12561) 2024-11-24T16:41:39,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742111_1287 (size=12151) 2024-11-24T16:41:39,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/34ba6f87891945cba45f25139a20978f 2024-11-24T16:41:39,302 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/ccfad34dc5084ad1a23b8348baef16a7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ccfad34dc5084ad1a23b8348baef16a7 2024-11-24T16:41:39,310 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into ccfad34dc5084ad1a23b8348baef16a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:39,310 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:39,310 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=13, startTime=1732466499160; duration=0sec 2024-11-24T16:41:39,310 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:39,310 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:39,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/074d65caf6aa414dbc45c58728c0d170 is 50, key is test_row_0/B:col10/1732466498211/Put/seqid=0 2024-11-24T16:41:39,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742112_1288 (size=12151) 2024-11-24T16:41:39,331 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/074d65caf6aa414dbc45c58728c0d170 2024-11-24T16:41:39,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/9024168278aa4bc280893fccc9370403 is 50, key is test_row_0/C:col10/1732466498211/Put/seqid=0 2024-11-24T16:41:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:39,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:39,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742113_1289 (size=12151) 2024-11-24T16:41:39,370 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/9024168278aa4bc280893fccc9370403 2024-11-24T16:41:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/34ba6f87891945cba45f25139a20978f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/34ba6f87891945cba45f25139a20978f 2024-11-24T16:41:39,380 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/34ba6f87891945cba45f25139a20978f, entries=150, sequenceid=259, filesize=11.9 K 2024-11-24T16:41:39,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/074d65caf6aa414dbc45c58728c0d170 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/074d65caf6aa414dbc45c58728c0d170 2024-11-24T16:41:39,384 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/074d65caf6aa414dbc45c58728c0d170, entries=150, sequenceid=259, filesize=11.9 K 2024-11-24T16:41:39,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/9024168278aa4bc280893fccc9370403 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9024168278aa4bc280893fccc9370403 2024-11-24T16:41:39,388 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9024168278aa4bc280893fccc9370403, entries=150, sequenceid=259, filesize=11.9 K 2024-11-24T16:41:39,401 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 200e097340e1bd260123b073b009f0f9 in 124ms, sequenceid=259, compaction requested=false 2024-11-24T16:41:39,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:39,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-24T16:41:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-24T16:41:39,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-24T16:41:39,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 283 msec 2024-11-24T16:41:39,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 290 msec 2024-11-24T16:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:39,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:41:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:39,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:39,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/23b671b7f0124aa29573952bad5848b5 is 50, key is test_row_0/A:col10/1732466499409/Put/seqid=0 2024-11-24T16:41:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T16:41:39,434 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:39,434 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-24T16:41:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T16:41:39,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:39,437 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:39,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:39,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742114_1290 (size=14741) 2024-11-24T16:41:39,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466559479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466499461, max timestamp=1732466499485Num uniques 859; 2024-11-24T16:41:39,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:39,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466499461, max timestamp=1732466499485Num uniques 859; 2024-11-24T16:41:39,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466499461, max timestamp=1732466499485Num uniques 859; 2024-11-24T16:41:39,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:39,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:41:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466559509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466559511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466559513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466559515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T16:41:39,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T16:41:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466559595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466559623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466559624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466559624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466559634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T16:41:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T16:41:39,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466559803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466559836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466559836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466559838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/23b671b7f0124aa29573952bad5848b5 2024-11-24T16:41:39,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466559843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:39,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:39,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/baabec57acae483b939aadcf8df7d269 is 50, key is test_row_0/B:col10/1732466499409/Put/seqid=0 2024-11-24T16:41:39,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742115_1291 (size=12301) 2024-11-24T16:41:39,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T16:41:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/baabec57acae483b939aadcf8df7d269 2024-11-24T16:41:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:39,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b25c3dcd914944449f4f917cb64a8a52 is 50, key is test_row_0/C:col10/1732466499409/Put/seqid=0 2024-11-24T16:41:39,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742116_1292 (size=12301) 2024-11-24T16:41:39,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b25c3dcd914944449f4f917cb64a8a52 2024-11-24T16:41:39,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/23b671b7f0124aa29573952bad5848b5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/23b671b7f0124aa29573952bad5848b5 2024-11-24T16:41:39,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/23b671b7f0124aa29573952bad5848b5, entries=200, sequenceid=272, filesize=14.4 K 2024-11-24T16:41:39,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/baabec57acae483b939aadcf8df7d269 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/baabec57acae483b939aadcf8df7d269 2024-11-24T16:41:39,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/baabec57acae483b939aadcf8df7d269, entries=150, sequenceid=272, filesize=12.0 K 2024-11-24T16:41:39,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b25c3dcd914944449f4f917cb64a8a52 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b25c3dcd914944449f4f917cb64a8a52 2024-11-24T16:41:39,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b25c3dcd914944449f4f917cb64a8a52, entries=150, sequenceid=272, filesize=12.0 K 2024-11-24T16:41:39,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 200e097340e1bd260123b073b009f0f9 in 572ms, sequenceid=272, compaction requested=true 2024-11-24T16:41:39,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:39,984 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:39,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:39,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:39,985 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:39,985 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:39,985 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:39,985 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,985 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/b14d77dd93fb4cd9ac243740b370316a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/34ba6f87891945cba45f25139a20978f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/23b671b7f0124aa29573952bad5848b5] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=38.6 K 2024-11-24T16:41:39,986 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b14d77dd93fb4cd9ac243740b370316a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732466497854 2024-11-24T16:41:39,986 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34ba6f87891945cba45f25139a20978f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732466498204 2024-11-24T16:41:39,987 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:39,987 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:39,987 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:39,987 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/ccc21e441b2e410db4508181110d190d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/074d65caf6aa414dbc45c58728c0d170, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/baabec57acae483b939aadcf8df7d269] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.2 K 2024-11-24T16:41:39,988 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc21e441b2e410db4508181110d190d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732466497854 2024-11-24T16:41:39,988 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b671b7f0124aa29573952bad5848b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732466499367 2024-11-24T16:41:39,988 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 074d65caf6aa414dbc45c58728c0d170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732466498204 2024-11-24T16:41:39,989 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting baabec57acae483b939aadcf8df7d269, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732466499367 2024-11-24T16:41:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:39,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:40,001 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:40,002 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/dca368f8cc3c4608acdb8c88b568b95a is 50, key is test_row_0/A:col10/1732466499409/Put/seqid=0 2024-11-24T16:41:40,005 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:40,006 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/a9e95d112aac4e9f8d0ee433f6183cec is 50, key is test_row_0/B:col10/1732466499409/Put/seqid=0 2024-11-24T16:41:40,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742117_1293 (size=12847) 2024-11-24T16:41:40,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T16:41:40,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:40,059 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-24T16:41:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:40,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:41:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T16:41:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742118_1294 (size=12847) 2024-11-24T16:41:40,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/e6e8d560e9844a9aa360540d968089ad is 50, key is test_row_0/A:col10/1732466499501/Put/seqid=0 2024-11-24T16:41:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:40,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:40,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742119_1295 (size=12301) 2024-11-24T16:41:40,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466560154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466560154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466560160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466560160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466560166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466560274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466560284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466560284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,463 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/dca368f8cc3c4608acdb8c88b568b95a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/dca368f8cc3c4608acdb8c88b568b95a 2024-11-24T16:41:40,472 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into dca368f8cc3c4608acdb8c88b568b95a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:40,472 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:40,472 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466499984; duration=0sec 2024-11-24T16:41:40,472 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:40,472 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:40,472 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:40,474 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:40,474 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:40,475 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:40,475 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ccfad34dc5084ad1a23b8348baef16a7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9024168278aa4bc280893fccc9370403, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b25c3dcd914944449f4f917cb64a8a52] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.1 K 2024-11-24T16:41:40,476 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccfad34dc5084ad1a23b8348baef16a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732466497854 2024-11-24T16:41:40,476 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9024168278aa4bc280893fccc9370403, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732466498204 2024-11-24T16:41:40,477 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b25c3dcd914944449f4f917cb64a8a52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732466499367 2024-11-24T16:41:40,487 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/a9e95d112aac4e9f8d0ee433f6183cec as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/a9e95d112aac4e9f8d0ee433f6183cec 2024-11-24T16:41:40,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,495 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into a9e95d112aac4e9f8d0ee433f6183cec(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:40,496 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:40,496 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466499985; duration=0sec 2024-11-24T16:41:40,496 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:40,496 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466560485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466560494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466560494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,504 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:40,505 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b7daf792e26e489697d8420252d79f12 is 50, key is test_row_0/C:col10/1732466499409/Put/seqid=0 2024-11-24T16:41:40,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742120_1296 (size=12813) 2024-11-24T16:41:40,522 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/e6e8d560e9844a9aa360540d968089ad 2024-11-24T16:41:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/1187868c5ed8470e8a7c7503c340a0b1 is 50, key is test_row_0/B:col10/1732466499501/Put/seqid=0 2024-11-24T16:41:40,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742121_1297 (size=12301) 2024-11-24T16:41:40,564 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/1187868c5ed8470e8a7c7503c340a0b1 2024-11-24T16:41:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T16:41:40,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/24ff71f5fd92477c86899b69d1f0ddee is 50, key is test_row_0/C:col10/1732466499501/Put/seqid=0 2024-11-24T16:41:40,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742122_1298 (size=12301) 2024-11-24T16:41:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466560675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466560684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,790 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T16:41:40,790 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T16:41:40,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466560803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466560803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466560814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:40,923 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/b7daf792e26e489697d8420252d79f12 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b7daf792e26e489697d8420252d79f12 2024-11-24T16:41:40,930 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into b7daf792e26e489697d8420252d79f12(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:40,930 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:40,930 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=13, startTime=1732466499998; duration=0sec 2024-11-24T16:41:40,930 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:40,930 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:41,011 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/24ff71f5fd92477c86899b69d1f0ddee 2024-11-24T16:41:41,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/e6e8d560e9844a9aa360540d968089ad as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e6e8d560e9844a9aa360540d968089ad 2024-11-24T16:41:41,030 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e6e8d560e9844a9aa360540d968089ad, entries=150, sequenceid=307, filesize=12.0 K 2024-11-24T16:41:41,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/1187868c5ed8470e8a7c7503c340a0b1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1187868c5ed8470e8a7c7503c340a0b1 2024-11-24T16:41:41,038 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1187868c5ed8470e8a7c7503c340a0b1, entries=150, sequenceid=307, filesize=12.0 K 2024-11-24T16:41:41,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/24ff71f5fd92477c86899b69d1f0ddee as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/24ff71f5fd92477c86899b69d1f0ddee 2024-11-24T16:41:41,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/24ff71f5fd92477c86899b69d1f0ddee, entries=150, sequenceid=307, filesize=12.0 K 2024-11-24T16:41:41,062 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for 200e097340e1bd260123b073b009f0f9 in 1003ms, sequenceid=307, compaction requested=false 2024-11-24T16:41:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-24T16:41:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-24T16:41:41,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T16:41:41,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-11-24T16:41:41,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6330 sec 2024-11-24T16:41:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:41,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:41:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:41,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0f2890f47f8746f087f67be7a066904c is 50, key is test_row_0/A:col10/1732466500123/Put/seqid=0 2024-11-24T16:41:41,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742123_1299 (size=14741) 2024-11-24T16:41:41,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0f2890f47f8746f087f67be7a066904c 2024-11-24T16:41:41,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/70e9a19f6cad46cbbf36c9a1e9558bef is 50, key is test_row_0/B:col10/1732466500123/Put/seqid=0 2024-11-24T16:41:41,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466561358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466561359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466561361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742124_1300 (size=12301) 2024-11-24T16:41:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/70e9a19f6cad46cbbf36c9a1e9558bef 2024-11-24T16:41:41,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/cc43561a5f9a40299bceb8efabd2f6b7 is 50, key is test_row_0/C:col10/1732466500123/Put/seqid=0 2024-11-24T16:41:41,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742125_1301 (size=12301) 2024-11-24T16:41:41,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466561473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466561474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466561474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T16:41:41,584 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:41,584 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-24T16:41:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T16:41:41,586 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:41,587 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:41,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:41,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466561683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466561684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466561684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T16:41:41,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466561694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,698 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466561703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:41,704 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:41,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T16:41:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:41,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:41,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:41,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/cc43561a5f9a40299bceb8efabd2f6b7 2024-11-24T16:41:41,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0f2890f47f8746f087f67be7a066904c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0f2890f47f8746f087f67be7a066904c 2024-11-24T16:41:41,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0f2890f47f8746f087f67be7a066904c, entries=200, sequenceid=324, filesize=14.4 K 2024-11-24T16:41:41,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/70e9a19f6cad46cbbf36c9a1e9558bef as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/70e9a19f6cad46cbbf36c9a1e9558bef 2024-11-24T16:41:41,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/70e9a19f6cad46cbbf36c9a1e9558bef, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T16:41:41,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/cc43561a5f9a40299bceb8efabd2f6b7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cc43561a5f9a40299bceb8efabd2f6b7 2024-11-24T16:41:41,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cc43561a5f9a40299bceb8efabd2f6b7, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T16:41:41,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 200e097340e1bd260123b073b009f0f9 in 489ms, sequenceid=324, compaction requested=true 2024-11-24T16:41:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:41,811 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:41,811 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:41,812 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:41,812 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:41,812 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:41,813 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/dca368f8cc3c4608acdb8c88b568b95a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e6e8d560e9844a9aa360540d968089ad, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0f2890f47f8746f087f67be7a066904c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=39.0 K 2024-11-24T16:41:41,813 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:41,813 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:41,813 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:41,813 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/a9e95d112aac4e9f8d0ee433f6183cec, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1187868c5ed8470e8a7c7503c340a0b1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/70e9a19f6cad46cbbf36c9a1e9558bef] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.6 K 2024-11-24T16:41:41,813 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca368f8cc3c4608acdb8c88b568b95a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732466499367 2024-11-24T16:41:41,814 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e95d112aac4e9f8d0ee433f6183cec, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732466499367 2024-11-24T16:41:41,815 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1187868c5ed8470e8a7c7503c340a0b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732466499501 2024-11-24T16:41:41,815 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e8d560e9844a9aa360540d968089ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732466499501 2024-11-24T16:41:41,815 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e9a19f6cad46cbbf36c9a1e9558bef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466500123 2024-11-24T16:41:41,815 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f2890f47f8746f087f67be7a066904c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466500123 2024-11-24T16:41:41,829 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:41,829 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/c021e5d7b5c145e2b07b16a81a52e9ea is 50, key is test_row_0/B:col10/1732466500123/Put/seqid=0 2024-11-24T16:41:41,830 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:41,831 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/2c8c9521bba94bd5b797847ba067dae9 is 50, key is test_row_0/A:col10/1732466500123/Put/seqid=0 2024-11-24T16:41:41,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742126_1302 (size=12949) 2024-11-24T16:41:41,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742127_1303 (size=12949) 2024-11-24T16:41:41,872 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/2c8c9521bba94bd5b797847ba067dae9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/2c8c9521bba94bd5b797847ba067dae9 2024-11-24T16:41:41,877 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 2c8c9521bba94bd5b797847ba067dae9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:41,877 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:41,877 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466501811; duration=0sec 2024-11-24T16:41:41,877 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:41,877 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:41,877 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:41,879 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:41,879 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:41,879 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:41,879 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b7daf792e26e489697d8420252d79f12, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/24ff71f5fd92477c86899b69d1f0ddee, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cc43561a5f9a40299bceb8efabd2f6b7] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.5 K 2024-11-24T16:41:41,880 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7daf792e26e489697d8420252d79f12, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732466499367 2024-11-24T16:41:41,880 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24ff71f5fd92477c86899b69d1f0ddee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732466499501 2024-11-24T16:41:41,881 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc43561a5f9a40299bceb8efabd2f6b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466500123 2024-11-24T16:41:41,889 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:41,890 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/ec29e306421e4367ace8905e68a835c9 is 50, key is test_row_0/C:col10/1732466500123/Put/seqid=0 2024-11-24T16:41:41,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T16:41:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:41,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T16:41:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:41,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:41,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742128_1304 (size=12915) 2024-11-24T16:41:41,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/a3e9d6b0420648c295195166ecd53726 is 50, key is test_row_0/A:col10/1732466501347/Put/seqid=0 2024-11-24T16:41:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T16:41:41,904 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/ec29e306421e4367ace8905e68a835c9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ec29e306421e4367ace8905e68a835c9 2024-11-24T16:41:41,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742129_1305 (size=12301) 2024-11-24T16:41:41,912 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into ec29e306421e4367ace8905e68a835c9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:41,912 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:41,912 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=13, startTime=1732466501811; duration=0sec 2024-11-24T16:41:41,912 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:41,912 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:41,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:42,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466562015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466562021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466562022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466562123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466562133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466562134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T16:41:42,264 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/c021e5d7b5c145e2b07b16a81a52e9ea as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c021e5d7b5c145e2b07b16a81a52e9ea 2024-11-24T16:41:42,270 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into c021e5d7b5c145e2b07b16a81a52e9ea(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:42,270 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:42,270 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466501811; duration=0sec 2024-11-24T16:41:42,270 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:42,270 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:42,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/a3e9d6b0420648c295195166ecd53726 2024-11-24T16:41:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e47903d89c8a4db3b5ba4996e02e99d0 is 50, key is test_row_0/B:col10/1732466501347/Put/seqid=0 2024-11-24T16:41:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466562334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742130_1306 (size=12301) 2024-11-24T16:41:42,346 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e47903d89c8a4db3b5ba4996e02e99d0 2024-11-24T16:41:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466562344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466562344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/c634e077350843e1b01d4d91c6f051ac is 50, key is test_row_0/C:col10/1732466501347/Put/seqid=0 2024-11-24T16:41:42,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742131_1307 (size=12301) 2024-11-24T16:41:42,378 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/c634e077350843e1b01d4d91c6f051ac 2024-11-24T16:41:42,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/a3e9d6b0420648c295195166ecd53726 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/a3e9d6b0420648c295195166ecd53726 2024-11-24T16:41:42,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/a3e9d6b0420648c295195166ecd53726, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T16:41:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e47903d89c8a4db3b5ba4996e02e99d0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e47903d89c8a4db3b5ba4996e02e99d0 2024-11-24T16:41:42,392 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e47903d89c8a4db3b5ba4996e02e99d0, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T16:41:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/c634e077350843e1b01d4d91c6f051ac as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/c634e077350843e1b01d4d91c6f051ac 2024-11-24T16:41:42,398 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/c634e077350843e1b01d4d91c6f051ac, entries=150, sequenceid=343, filesize=12.0 K 2024-11-24T16:41:42,399 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 200e097340e1bd260123b073b009f0f9 in 506ms, sequenceid=343, compaction requested=false 2024-11-24T16:41:42,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-24T16:41:42,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-24T16:41:42,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-24T16:41:42,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 815 msec 2024-11-24T16:41:42,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 823 msec 2024-11-24T16:41:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:42,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T16:41:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:42,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0bceff5da9504f0bb713b8ea435232ea is 50, key is test_row_0/A:col10/1732466502020/Put/seqid=0 2024-11-24T16:41:42,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466562676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742132_1308 (size=14741) 2024-11-24T16:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466562678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0bceff5da9504f0bb713b8ea435232ea 2024-11-24T16:41:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466562680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e2721d30d1d34b589d3db698806c5bb1 is 50, key is test_row_0/B:col10/1732466502020/Put/seqid=0 2024-11-24T16:41:42,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742133_1309 (size=12301) 2024-11-24T16:41:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T16:41:42,724 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:42,725 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-24T16:41:42,727 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T16:41:42,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:42,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:42,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466562794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466562794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466562794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T16:41:42,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T16:41:42,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:42,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:42,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:42,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:42,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466563003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466563003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466563003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T16:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T16:41:43,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e2721d30d1d34b589d3db698806c5bb1 2024-11-24T16:41:43,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/581681c638d84f33bea77b70a53cfbf8 is 50, key is test_row_0/C:col10/1732466502020/Put/seqid=0 2024-11-24T16:41:43,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742134_1310 (size=12301) 2024-11-24T16:41:43,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T16:41:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466563313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466563314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466563314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T16:41:43,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:43,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T16:41:43,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T16:41:43,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:43,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:43,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/581681c638d84f33bea77b70a53cfbf8 2024-11-24T16:41:43,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0bceff5da9504f0bb713b8ea435232ea as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0bceff5da9504f0bb713b8ea435232ea 2024-11-24T16:41:43,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0bceff5da9504f0bb713b8ea435232ea, entries=200, sequenceid=361, filesize=14.4 K 2024-11-24T16:41:43,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/e2721d30d1d34b589d3db698806c5bb1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e2721d30d1d34b589d3db698806c5bb1 2024-11-24T16:41:43,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e2721d30d1d34b589d3db698806c5bb1, entries=150, sequenceid=361, filesize=12.0 K 2024-11-24T16:41:43,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/581681c638d84f33bea77b70a53cfbf8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/581681c638d84f33bea77b70a53cfbf8 2024-11-24T16:41:43,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/581681c638d84f33bea77b70a53cfbf8, entries=150, sequenceid=361, filesize=12.0 K 2024-11-24T16:41:43,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 200e097340e1bd260123b073b009f0f9 in 929ms, sequenceid=361, compaction requested=true 2024-11-24T16:41:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:43,577 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:43,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:43,577 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:43,580 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:43,580 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:43,580 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,580 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/2c8c9521bba94bd5b797847ba067dae9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/a3e9d6b0420648c295195166ecd53726, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0bceff5da9504f0bb713b8ea435232ea] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=39.1 K 2024-11-24T16:41:43,580 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:43,580 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:43,580 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,580 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c021e5d7b5c145e2b07b16a81a52e9ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e47903d89c8a4db3b5ba4996e02e99d0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e2721d30d1d34b589d3db698806c5bb1] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.7 K 2024-11-24T16:41:43,581 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c021e5d7b5c145e2b07b16a81a52e9ea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466500123 2024-11-24T16:41:43,581 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c8c9521bba94bd5b797847ba067dae9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466500123 2024-11-24T16:41:43,582 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e47903d89c8a4db3b5ba4996e02e99d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732466501346 2024-11-24T16:41:43,583 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e9d6b0420648c295195166ecd53726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732466501346 2024-11-24T16:41:43,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e2721d30d1d34b589d3db698806c5bb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732466502013 2024-11-24T16:41:43,583 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bceff5da9504f0bb713b8ea435232ea, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732466502013 2024-11-24T16:41:43,602 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:43,602 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/92546cadf3d348469adf56a42dffac79 is 50, key is test_row_0/B:col10/1732466502020/Put/seqid=0 2024-11-24T16:41:43,610 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:43,611 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/75533a3950314b14b6f6ae35efe28ea5 is 50, key is test_row_0/A:col10/1732466502020/Put/seqid=0 2024-11-24T16:41:43,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742135_1311 (size=13051) 2024-11-24T16:41:43,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742136_1312 (size=13051) 2024-11-24T16:41:43,648 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/75533a3950314b14b6f6ae35efe28ea5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/75533a3950314b14b6f6ae35efe28ea5 2024-11-24T16:41:43,654 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 75533a3950314b14b6f6ae35efe28ea5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:43,654 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:43,654 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466503577; duration=0sec 2024-11-24T16:41:43,654 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:43,654 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:43,654 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:43,655 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:43,655 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:43,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T16:41:43,655 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,656 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ec29e306421e4367ace8905e68a835c9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/c634e077350843e1b01d4d91c6f051ac, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/581681c638d84f33bea77b70a53cfbf8] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.6 K 2024-11-24T16:41:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,656 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T16:41:43,656 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec29e306421e4367ace8905e68a835c9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732466500123 2024-11-24T16:41:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:43,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:43,658 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting c634e077350843e1b01d4d91c6f051ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732466501346 2024-11-24T16:41:43,658 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 581681c638d84f33bea77b70a53cfbf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732466502013 2024-11-24T16:41:43,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/822dbbcb8d4241fb811910472285ef27 is 50, key is test_row_0/A:col10/1732466502664/Put/seqid=0 2024-11-24T16:41:43,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742137_1313 (size=12301) 2024-11-24T16:41:43,668 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/822dbbcb8d4241fb811910472285ef27 2024-11-24T16:41:43,671 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:43,672 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/62f2ca9df42e412db770f0b5bf38b159 is 50, key is test_row_0/C:col10/1732466502020/Put/seqid=0 2024-11-24T16:41:43,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/c0f2d9c2725f483f91832d4f40a9efb7 is 50, key is test_row_0/B:col10/1732466502664/Put/seqid=0 2024-11-24T16:41:43,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742138_1314 (size=13017) 2024-11-24T16:41:43,689 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/62f2ca9df42e412db770f0b5bf38b159 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/62f2ca9df42e412db770f0b5bf38b159 2024-11-24T16:41:43,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742139_1315 (size=12301) 2024-11-24T16:41:43,703 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/c0f2d9c2725f483f91832d4f40a9efb7 2024-11-24T16:41:43,704 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into 62f2ca9df42e412db770f0b5bf38b159(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:43,704 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:43,704 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=13, startTime=1732466503577; duration=0sec 2024-11-24T16:41:43,704 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:43,704 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:43,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:43,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:43,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/7f02fd8e0d8943fca4ef9f0567760b80 is 50, key is test_row_0/C:col10/1732466502664/Put/seqid=0 2024-11-24T16:41:43,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466563760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466563761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742140_1316 (size=12301) 2024-11-24T16:41:43,771 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/7f02fd8e0d8943fca4ef9f0567760b80 2024-11-24T16:41:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/822dbbcb8d4241fb811910472285ef27 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/822dbbcb8d4241fb811910472285ef27 2024-11-24T16:41:43,781 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/822dbbcb8d4241fb811910472285ef27, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T16:41:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/c0f2d9c2725f483f91832d4f40a9efb7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c0f2d9c2725f483f91832d4f40a9efb7 2024-11-24T16:41:43,788 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c0f2d9c2725f483f91832d4f40a9efb7, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T16:41:43,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/7f02fd8e0d8943fca4ef9f0567760b80 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7f02fd8e0d8943fca4ef9f0567760b80 2024-11-24T16:41:43,793 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7f02fd8e0d8943fca4ef9f0567760b80, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T16:41:43,796 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 200e097340e1bd260123b073b009f0f9 in 140ms, sequenceid=378, compaction requested=false 2024-11-24T16:41:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-24T16:41:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-24T16:41:43,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T16:41:43,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-24T16:41:43,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0770 sec 2024-11-24T16:41:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:43,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T16:41:43,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:43,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:43,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:43,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:43,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:43,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:43,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/ab75d10be8d147c8a54e1c562ba3f87c is 50, key is test_row_0/A:col10/1732466503826/Put/seqid=0 2024-11-24T16:41:43,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742141_1317 (size=14741) 2024-11-24T16:41:43,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466563852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466563853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466563857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T16:41:43,864 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:43,864 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:43,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-24T16:41:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T16:41:43,867 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:43,868 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:43,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:43,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466563874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466563874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466563964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466563964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:43,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T16:41:43,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466563973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:43,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:44,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,041 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/92546cadf3d348469adf56a42dffac79 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/92546cadf3d348469adf56a42dffac79 2024-11-24T16:41:44,046 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into 92546cadf3d348469adf56a42dffac79(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:44,046 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:44,046 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466503577; duration=0sec 2024-11-24T16:41:44,046 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:44,046 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:44,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466564083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466564084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466564173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466564174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T16:41:44,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466564184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/ab75d10be8d147c8a54e1c562ba3f87c 2024-11-24T16:41:44,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/422e96fbb48a4b15a2200ad356eb3055 is 50, key is test_row_0/B:col10/1732466503826/Put/seqid=0 2024-11-24T16:41:44,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742142_1318 (size=12301) 2024-11-24T16:41:44,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:44,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466564394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466564394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:44,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466564484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466564485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T16:41:44,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466564494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/422e96fbb48a4b15a2200ad356eb3055 2024-11-24T16:41:44,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/cd6a1ec313b04cd6a0e625675ddafebc is 50, key is test_row_0/C:col10/1732466503826/Put/seqid=0 2024-11-24T16:41:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742143_1319 (size=12301) 2024-11-24T16:41:44,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466564904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466564904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:44,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:44,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466564994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466564994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T16:41:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:45,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466565003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:45,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:45,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/cd6a1ec313b04cd6a0e625675ddafebc 2024-11-24T16:41:45,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/ab75d10be8d147c8a54e1c562ba3f87c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab75d10be8d147c8a54e1c562ba3f87c 2024-11-24T16:41:45,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab75d10be8d147c8a54e1c562ba3f87c, entries=200, sequenceid=397, filesize=14.4 K 2024-11-24T16:41:45,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/422e96fbb48a4b15a2200ad356eb3055 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/422e96fbb48a4b15a2200ad356eb3055 2024-11-24T16:41:45,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/422e96fbb48a4b15a2200ad356eb3055, entries=150, sequenceid=397, filesize=12.0 K 2024-11-24T16:41:45,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/cd6a1ec313b04cd6a0e625675ddafebc as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cd6a1ec313b04cd6a0e625675ddafebc 2024-11-24T16:41:45,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:45,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:45,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:45,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:45,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:45,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:45,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cd6a1ec313b04cd6a0e625675ddafebc, entries=150, sequenceid=397, filesize=12.0 K 2024-11-24T16:41:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:45,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 200e097340e1bd260123b073b009f0f9 in 1277ms, sequenceid=397, compaction requested=true 2024-11-24T16:41:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:45,106 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:45,106 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:45,107 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:45,107 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:45,107 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:45,107 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/75533a3950314b14b6f6ae35efe28ea5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/822dbbcb8d4241fb811910472285ef27, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab75d10be8d147c8a54e1c562ba3f87c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=39.2 K 2024-11-24T16:41:45,108 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75533a3950314b14b6f6ae35efe28ea5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732466502013 2024-11-24T16:41:45,108 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 822dbbcb8d4241fb811910472285ef27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732466502664 2024-11-24T16:41:45,109 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab75d10be8d147c8a54e1c562ba3f87c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466503748 2024-11-24T16:41:45,112 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:45,112 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:45,112 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:45,112 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/92546cadf3d348469adf56a42dffac79, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c0f2d9c2725f483f91832d4f40a9efb7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/422e96fbb48a4b15a2200ad356eb3055] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.8 K 2024-11-24T16:41:45,113 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 92546cadf3d348469adf56a42dffac79, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732466502013 2024-11-24T16:41:45,114 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c0f2d9c2725f483f91832d4f40a9efb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732466502664 2024-11-24T16:41:45,114 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 422e96fbb48a4b15a2200ad356eb3055, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466503757 2024-11-24T16:41:45,117 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:45,117 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/6760267e07be462191a35eb6c9a8dc6c is 50, key is test_row_0/A:col10/1732466503826/Put/seqid=0 2024-11-24T16:41:45,124 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:45,124 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/c3664720067d46d1b68c35978a0f504b is 50, key is test_row_0/B:col10/1732466503826/Put/seqid=0 2024-11-24T16:41:45,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742144_1320 (size=13153) 2024-11-24T16:41:45,151 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/6760267e07be462191a35eb6c9a8dc6c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6760267e07be462191a35eb6c9a8dc6c 2024-11-24T16:41:45,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742145_1321 (size=13153) 2024-11-24T16:41:45,157 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 6760267e07be462191a35eb6c9a8dc6c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:45,157 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:45,157 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466505106; duration=0sec 2024-11-24T16:41:45,157 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:45,157 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:45,157 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:45,159 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:45,159 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:45,159 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:45,159 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/62f2ca9df42e412db770f0b5bf38b159, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7f02fd8e0d8943fca4ef9f0567760b80, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cd6a1ec313b04cd6a0e625675ddafebc] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.7 K 2024-11-24T16:41:45,159 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f2ca9df42e412db770f0b5bf38b159, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1732466502013 2024-11-24T16:41:45,160 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f02fd8e0d8943fca4ef9f0567760b80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1732466502664 2024-11-24T16:41:45,161 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6a1ec313b04cd6a0e625675ddafebc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466503757 2024-11-24T16:41:45,161 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/c3664720067d46d1b68c35978a0f504b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c3664720067d46d1b68c35978a0f504b 2024-11-24T16:41:45,167 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into c3664720067d46d1b68c35978a0f504b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:45,167 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:45,167 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466505106; duration=0sec 2024-11-24T16:41:45,167 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:45,167 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:45,181 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#277 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:45,182 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/3bc255eaf8a2486c887ee4fa4131677d is 50, key is test_row_0/C:col10/1732466503826/Put/seqid=0 2024-11-24T16:41:45,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742146_1322 (size=13119) 2024-11-24T16:41:45,218 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/3bc255eaf8a2486c887ee4fa4131677d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3bc255eaf8a2486c887ee4fa4131677d 2024-11-24T16:41:45,226 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into 3bc255eaf8a2486c887ee4fa4131677d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:45,226 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:45,226 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=13, startTime=1732466505106; duration=0sec 2024-11-24T16:41:45,226 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:45,226 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:45,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T16:41:45,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:45,257 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T16:41:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:45,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0e1d3e0a25484684ad68d39c948d2dee is 50, key is test_row_0/A:col10/1732466503855/Put/seqid=0 2024-11-24T16:41:45,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742147_1323 (size=12301) 2024-11-24T16:41:45,669 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0e1d3e0a25484684ad68d39c948d2dee 2024-11-24T16:41:45,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/6a8bad7eb148416f8a7ec565f2608913 is 50, key is test_row_0/B:col10/1732466503855/Put/seqid=0 2024-11-24T16:41:45,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742148_1324 (size=12301) 2024-11-24T16:41:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:45,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:45,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466565954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:45,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:45,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466565955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:45,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466566004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,005 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T16:41:46,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466566013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,014 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466566013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,017 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466566064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466566064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,081 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/6a8bad7eb148416f8a7ec565f2608913 2024-11-24T16:41:46,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/6e387106620d401d854d18df4ca53674 is 50, key is test_row_0/C:col10/1732466503855/Put/seqid=0 2024-11-24T16:41:46,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742149_1325 (size=12301) 2024-11-24T16:41:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466566273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466566274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,493 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/6e387106620d401d854d18df4ca53674 2024-11-24T16:41:46,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0e1d3e0a25484684ad68d39c948d2dee as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0e1d3e0a25484684ad68d39c948d2dee 2024-11-24T16:41:46,502 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0e1d3e0a25484684ad68d39c948d2dee, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T16:41:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/6a8bad7eb148416f8a7ec565f2608913 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6a8bad7eb148416f8a7ec565f2608913 2024-11-24T16:41:46,507 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6a8bad7eb148416f8a7ec565f2608913, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T16:41:46,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/6e387106620d401d854d18df4ca53674 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/6e387106620d401d854d18df4ca53674 2024-11-24T16:41:46,511 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/6e387106620d401d854d18df4ca53674, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T16:41:46,512 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 200e097340e1bd260123b073b009f0f9 in 1255ms, sequenceid=415, compaction requested=false 2024-11-24T16:41:46,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:46,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:46,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-24T16:41:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-24T16:41:46,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T16:41:46,517 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6460 sec 2024-11-24T16:41:46,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.6530 sec 2024-11-24T16:41:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:46,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T16:41:46,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:46,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:46,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:46,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:46,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:46,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:46,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/303084921e754a41a30179a7fca9510c is 50, key is test_row_0/A:col10/1732466505937/Put/seqid=0 2024-11-24T16:41:46,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742150_1326 (size=12301) 2024-11-24T16:41:46,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466566609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466566610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466566713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466566723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466566924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466566934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:46,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:46,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/303084921e754a41a30179a7fca9510c 2024-11-24T16:41:47,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/270f8145b58d4c7fab78c55b77fa81d2 is 50, key is test_row_0/B:col10/1732466505937/Put/seqid=0 2024-11-24T16:41:47,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742151_1327 (size=12301) 2024-11-24T16:41:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466567233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466567243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:47,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/270f8145b58d4c7fab78c55b77fa81d2 2024-11-24T16:41:47,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/fa2f9830e90a418eb2b0e5fdfe2d3b67 is 50, key is test_row_0/C:col10/1732466505937/Put/seqid=0 2024-11-24T16:41:47,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742152_1328 (size=12301) 2024-11-24T16:41:47,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466567744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466567753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:47,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/fa2f9830e90a418eb2b0e5fdfe2d3b67 2024-11-24T16:41:47,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/303084921e754a41a30179a7fca9510c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/303084921e754a41a30179a7fca9510c 2024-11-24T16:41:47,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/303084921e754a41a30179a7fca9510c, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T16:41:47,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/270f8145b58d4c7fab78c55b77fa81d2 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/270f8145b58d4c7fab78c55b77fa81d2 2024-11-24T16:41:47,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/270f8145b58d4c7fab78c55b77fa81d2, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T16:41:47,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/fa2f9830e90a418eb2b0e5fdfe2d3b67 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/fa2f9830e90a418eb2b0e5fdfe2d3b67 2024-11-24T16:41:47,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/fa2f9830e90a418eb2b0e5fdfe2d3b67, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T16:41:47,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 200e097340e1bd260123b073b009f0f9 in 1263ms, sequenceid=434, compaction requested=true 2024-11-24T16:41:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:47,849 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:47,849 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:47,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:47,850 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:47,850 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:47,850 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:47,850 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:47,850 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:47,850 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:47,851 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c3664720067d46d1b68c35978a0f504b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6a8bad7eb148416f8a7ec565f2608913, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/270f8145b58d4c7fab78c55b77fa81d2] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.9 K 2024-11-24T16:41:47,851 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6760267e07be462191a35eb6c9a8dc6c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0e1d3e0a25484684ad68d39c948d2dee, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/303084921e754a41a30179a7fca9510c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.9 K 2024-11-24T16:41:47,851 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6760267e07be462191a35eb6c9a8dc6c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466503757 2024-11-24T16:41:47,851 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c3664720067d46d1b68c35978a0f504b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466503757 2024-11-24T16:41:47,851 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1d3e0a25484684ad68d39c948d2dee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732466503842 2024-11-24T16:41:47,851 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8bad7eb148416f8a7ec565f2608913, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732466503842 2024-11-24T16:41:47,852 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 270f8145b58d4c7fab78c55b77fa81d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732466505937 2024-11-24T16:41:47,852 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 303084921e754a41a30179a7fca9510c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732466505937 2024-11-24T16:41:47,860 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:47,860 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/15498381706b4dbdb3d7c0a7867271c8 is 50, key is test_row_0/B:col10/1732466505937/Put/seqid=0 2024-11-24T16:41:47,860 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:47,861 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/1f6e29dad2b44e799b22ccd4e2d987b0 is 50, key is test_row_0/A:col10/1732466505937/Put/seqid=0 2024-11-24T16:41:47,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742153_1329 (size=13255) 2024-11-24T16:41:47,876 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/15498381706b4dbdb3d7c0a7867271c8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/15498381706b4dbdb3d7c0a7867271c8 2024-11-24T16:41:47,885 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into 15498381706b4dbdb3d7c0a7867271c8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:47,885 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:47,885 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466507849; duration=0sec 2024-11-24T16:41:47,885 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:47,885 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:47,885 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:47,887 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:47,887 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:47,887 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:47,887 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3bc255eaf8a2486c887ee4fa4131677d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/6e387106620d401d854d18df4ca53674, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/fa2f9830e90a418eb2b0e5fdfe2d3b67] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=36.8 K 2024-11-24T16:41:47,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742154_1330 (size=13255) 2024-11-24T16:41:47,888 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc255eaf8a2486c887ee4fa4131677d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466503757 2024-11-24T16:41:47,889 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e387106620d401d854d18df4ca53674, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732466503842 2024-11-24T16:41:47,889 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fa2f9830e90a418eb2b0e5fdfe2d3b67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732466505937 2024-11-24T16:41:47,896 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:47,897 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/3ece7223e8444516aac1ab2bcf6e2e76 is 50, key is test_row_0/C:col10/1732466505937/Put/seqid=0 2024-11-24T16:41:47,899 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/1f6e29dad2b44e799b22ccd4e2d987b0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1f6e29dad2b44e799b22ccd4e2d987b0 2024-11-24T16:41:47,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742155_1331 (size=13221) 2024-11-24T16:41:47,905 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 1f6e29dad2b44e799b22ccd4e2d987b0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:47,905 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:47,905 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466507849; duration=0sec 2024-11-24T16:41:47,905 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:47,905 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:47,909 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/3ece7223e8444516aac1ab2bcf6e2e76 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3ece7223e8444516aac1ab2bcf6e2e76 2024-11-24T16:41:47,914 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into 3ece7223e8444516aac1ab2bcf6e2e76(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:47,914 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:47,914 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=13, startTime=1732466507849; duration=0sec 2024-11-24T16:41:47,914 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:47,914 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:48,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:48,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T16:41:48,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:48,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:48,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:48,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:48,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:48,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:48,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/e4a3b011ef3d435e9f630361ee832890 is 50, key is test_row_0/A:col10/1732466506607/Put/seqid=0 2024-11-24T16:41:48,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742156_1332 (size=14741) 2024-11-24T16:41:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T16:41:48,034 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:48,035 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-24T16:41:48,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T16:41:48,036 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:48,037 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:48,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466568047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466568048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466568052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T16:41:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466568163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466568163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466568163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T16:41:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466568374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466568374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466568374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/e4a3b011ef3d435e9f630361ee832890 2024-11-24T16:41:48,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/0b12db530d3a482eb925d49c04fdaca9 is 50, key is test_row_0/B:col10/1732466506607/Put/seqid=0 2024-11-24T16:41:48,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742157_1333 (size=12301) 2024-11-24T16:41:48,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:48,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:48,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T16:41:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466568683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466568684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466568684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466568763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,765 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466568773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:48,777 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:48,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:48,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/0b12db530d3a482eb925d49c04fdaca9 2024-11-24T16:41:48,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/56976f3f6c3a467db6567951f6122397 is 50, key is test_row_0/C:col10/1732466506607/Put/seqid=0 2024-11-24T16:41:48,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742158_1334 (size=12301) 2024-11-24T16:41:48,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:48,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:49,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:49,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:49,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:49,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:49,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:49,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:49,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T16:41:49,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466569194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:49,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466569194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:49,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:49,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466569194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:49,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/56976f3f6c3a467db6567951f6122397 2024-11-24T16:41:49,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/e4a3b011ef3d435e9f630361ee832890 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e4a3b011ef3d435e9f630361ee832890 2024-11-24T16:41:49,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e4a3b011ef3d435e9f630361ee832890, entries=200, sequenceid=451, filesize=14.4 K 2024-11-24T16:41:49,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/0b12db530d3a482eb925d49c04fdaca9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/0b12db530d3a482eb925d49c04fdaca9 2024-11-24T16:41:49,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/0b12db530d3a482eb925d49c04fdaca9, entries=150, sequenceid=451, filesize=12.0 K 2024-11-24T16:41:49,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/56976f3f6c3a467db6567951f6122397 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/56976f3f6c3a467db6567951f6122397 2024-11-24T16:41:49,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/56976f3f6c3a467db6567951f6122397, entries=150, sequenceid=451, filesize=12.0 K 2024-11-24T16:41:49,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 200e097340e1bd260123b073b009f0f9 in 1259ms, sequenceid=451, compaction requested=false 2024-11-24T16:41:49,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:49,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T16:41:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:49,277 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T16:41:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:49,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:49,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0eba0f3783804e11ae528cc09175157c is 50, key is test_row_0/A:col10/1732466508050/Put/seqid=0 2024-11-24T16:41:49,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742159_1335 (size=12301) 2024-11-24T16:41:49,286 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0eba0f3783804e11ae528cc09175157c 2024-11-24T16:41:49,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/898f5b2eeb7f48e492d8a46b4e17653e is 50, key is test_row_0/B:col10/1732466508050/Put/seqid=0 2024-11-24T16:41:49,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742160_1336 (size=12301) 2024-11-24T16:41:49,298 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/898f5b2eeb7f48e492d8a46b4e17653e 2024-11-24T16:41:49,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a024431c2eec482281775138771e1688 is 50, key is test_row_0/C:col10/1732466508050/Put/seqid=0 2024-11-24T16:41:49,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742161_1337 (size=12301) 2024-11-24T16:41:49,710 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a024431c2eec482281775138771e1688 2024-11-24T16:41:49,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0eba0f3783804e11ae528cc09175157c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0eba0f3783804e11ae528cc09175157c 2024-11-24T16:41:49,719 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0eba0f3783804e11ae528cc09175157c, entries=150, sequenceid=469, filesize=12.0 K 2024-11-24T16:41:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/898f5b2eeb7f48e492d8a46b4e17653e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/898f5b2eeb7f48e492d8a46b4e17653e 2024-11-24T16:41:49,723 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/898f5b2eeb7f48e492d8a46b4e17653e, entries=150, sequenceid=469, filesize=12.0 K 2024-11-24T16:41:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a024431c2eec482281775138771e1688 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a024431c2eec482281775138771e1688 2024-11-24T16:41:49,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a024431c2eec482281775138771e1688, entries=150, sequenceid=469, filesize=12.0 K 2024-11-24T16:41:49,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 200e097340e1bd260123b073b009f0f9 in 452ms, sequenceid=469, compaction requested=true 2024-11-24T16:41:49,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:49,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:49,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-24T16:41:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-24T16:41:49,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-24T16:41:49,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6930 sec 2024-11-24T16:41:49,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.6980 sec 2024-11-24T16:41:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T16:41:50,184 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:50,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:41:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-24T16:41:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T16:41:50,187 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:41:50,188 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:41:50,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:41:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:50,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:41:50,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:50,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:50,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:50,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:50,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:50,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:50,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/ab9614de5a7a4a7c937372adff804acb is 50, key is test_row_0/A:col10/1732466510206/Put/seqid=0 2024-11-24T16:41:50,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742162_1338 (size=17181) 2024-11-24T16:41:50,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/ab9614de5a7a4a7c937372adff804acb 2024-11-24T16:41:50,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/d83da353eb1d4b52b1849325366c8b24 is 50, key is test_row_0/B:col10/1732466510206/Put/seqid=0 2024-11-24T16:41:50,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742163_1339 (size=12301) 2024-11-24T16:41:50,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466570251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466570253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466570255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T16:41:50,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T16:41:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466570364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466570364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466570364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T16:41:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T16:41:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466570574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466570574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466570574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/d83da353eb1d4b52b1849325366c8b24 2024-11-24T16:41:50,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T16:41:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a97a21bef74c4d3b811bbf75cff85089 is 50, key is test_row_0/C:col10/1732466510206/Put/seqid=0 2024-11-24T16:41:50,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742164_1340 (size=12301) 2024-11-24T16:41:50,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49218 deadline: 1732466570774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,779 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49212 deadline: 1732466570794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,799 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 200e097340e1bd260123b073b009f0f9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T16:41:50,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:50,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T16:41:50,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466570884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466570884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466570884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:50,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T16:41:50,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:50,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:50,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:41:51,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a97a21bef74c4d3b811bbf75cff85089 2024-11-24T16:41:51,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/ab9614de5a7a4a7c937372adff804acb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab9614de5a7a4a7c937372adff804acb 2024-11-24T16:41:51,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab9614de5a7a4a7c937372adff804acb, entries=250, sequenceid=481, filesize=16.8 K 2024-11-24T16:41:51,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/d83da353eb1d4b52b1849325366c8b24 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d83da353eb1d4b52b1849325366c8b24 2024-11-24T16:41:51,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d83da353eb1d4b52b1849325366c8b24, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T16:41:51,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/a97a21bef74c4d3b811bbf75cff85089 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a97a21bef74c4d3b811bbf75cff85089 2024-11-24T16:41:51,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a97a21bef74c4d3b811bbf75cff85089, entries=150, sequenceid=481, filesize=12.0 K 2024-11-24T16:41:51,076 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-24T16:41:51,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 200e097340e1bd260123b073b009f0f9 in 862ms, sequenceid=481, compaction requested=true 2024-11-24T16:41:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:51,077 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:51,077 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:51,078 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57478 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:51,078 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:51,078 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:51,078 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1f6e29dad2b44e799b22ccd4e2d987b0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e4a3b011ef3d435e9f630361ee832890, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0eba0f3783804e11ae528cc09175157c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab9614de5a7a4a7c937372adff804acb] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=56.1 K 2024-11-24T16:41:51,079 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:51,079 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f6e29dad2b44e799b22ccd4e2d987b0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732466505937 2024-11-24T16:41:51,079 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:51,079 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:51,079 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/15498381706b4dbdb3d7c0a7867271c8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/0b12db530d3a482eb925d49c04fdaca9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/898f5b2eeb7f48e492d8a46b4e17653e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d83da353eb1d4b52b1849325366c8b24] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=49.0 K 2024-11-24T16:41:51,079 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4a3b011ef3d435e9f630361ee832890, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732466506594 2024-11-24T16:41:51,079 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eba0f3783804e11ae528cc09175157c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732466508045 2024-11-24T16:41:51,079 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 15498381706b4dbdb3d7c0a7867271c8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732466505937 2024-11-24T16:41:51,080 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b12db530d3a482eb925d49c04fdaca9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732466506607 2024-11-24T16:41:51,080 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9614de5a7a4a7c937372adff804acb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466510206 2024-11-24T16:41:51,080 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 898f5b2eeb7f48e492d8a46b4e17653e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732466508045 2024-11-24T16:41:51,081 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d83da353eb1d4b52b1849325366c8b24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466510206 2024-11-24T16:41:51,088 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:51,089 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0d2cb612b38f4f58ba4fd5f3c2e209e0 is 50, key is test_row_0/A:col10/1732466510206/Put/seqid=0 2024-11-24T16:41:51,092 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:51,092 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/d3940bfca46a49268f2541235f3a4a63 is 50, key is test_row_0/B:col10/1732466510206/Put/seqid=0 2024-11-24T16:41:51,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742165_1341 (size=13391) 2024-11-24T16:41:51,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742166_1342 (size=13391) 2024-11-24T16:41:51,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T16:41:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:51,106 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:41:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:51,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/f5c1e01fd4db44c982f123619636b74e is 50, key is test_row_0/A:col10/1732466510243/Put/seqid=0 2024-11-24T16:41:51,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742167_1343 (size=12301) 2024-11-24T16:41:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T16:41:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:51,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. as already flushing 2024-11-24T16:41:51,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466571406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466571412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466571413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,503 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/0d2cb612b38f4f58ba4fd5f3c2e209e0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0d2cb612b38f4f58ba4fd5f3c2e209e0 2024-11-24T16:41:51,508 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 0d2cb612b38f4f58ba4fd5f3c2e209e0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:51,508 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:51,508 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=12, startTime=1732466511077; duration=0sec 2024-11-24T16:41:51,508 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:51,508 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:51,508 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:41:51,509 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:41:51,509 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:51,509 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:51,510 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3ece7223e8444516aac1ab2bcf6e2e76, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/56976f3f6c3a467db6567951f6122397, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a024431c2eec482281775138771e1688, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a97a21bef74c4d3b811bbf75cff85089] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=48.9 K 2024-11-24T16:41:51,510 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/d3940bfca46a49268f2541235f3a4a63 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d3940bfca46a49268f2541235f3a4a63 2024-11-24T16:41:51,510 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ece7223e8444516aac1ab2bcf6e2e76, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732466505937 2024-11-24T16:41:51,510 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56976f3f6c3a467db6567951f6122397, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1732466506607 2024-11-24T16:41:51,511 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a024431c2eec482281775138771e1688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1732466508045 2024-11-24T16:41:51,511 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97a21bef74c4d3b811bbf75cff85089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466510206 2024-11-24T16:41:51,515 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/f5c1e01fd4db44c982f123619636b74e 2024-11-24T16:41:51,517 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into d3940bfca46a49268f2541235f3a4a63(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:51,517 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:51,517 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=12, startTime=1732466511077; duration=0sec 2024-11-24T16:41:51,517 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:51,517 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:51,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/3ba55ca3ec1d4c9680805bc634df958e is 50, key is test_row_0/B:col10/1732466510243/Put/seqid=0 2024-11-24T16:41:51,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466571524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466571524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,526 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#300 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466571524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,526 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,527 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/d74db40055fd4cb4b2c2b677c43c1504 is 50, key is test_row_0/C:col10/1732466510206/Put/seqid=0 2024-11-24T16:41:51,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742169_1345 (size=13357) 2024-11-24T16:41:51,537 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/d74db40055fd4cb4b2c2b677c43c1504 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d74db40055fd4cb4b2c2b677c43c1504 2024-11-24T16:41:51,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742168_1344 (size=12301) 2024-11-24T16:41:51,543 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into d74db40055fd4cb4b2c2b677c43c1504(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:51,543 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:51,543 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=12, startTime=1732466511077; duration=0sec 2024-11-24T16:41:51,543 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:51,543 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466571734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466571734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466571734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:51,943 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/3ba55ca3ec1d4c9680805bc634df958e 2024-11-24T16:41:51,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/ac3020527b184cd5b67ab35e6b0a99a9 is 50, key is test_row_0/C:col10/1732466510243/Put/seqid=0 2024-11-24T16:41:51,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742170_1346 (size=12301) 2024-11-24T16:41:52,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49226 deadline: 1732466572043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49256 deadline: 1732466572043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49174 deadline: 1732466572044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:41:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:41:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T16:41:52,355 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/ac3020527b184cd5b67ab35e6b0a99a9 2024-11-24T16:41:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/f5c1e01fd4db44c982f123619636b74e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5c1e01fd4db44c982f123619636b74e 2024-11-24T16:41:52,364 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5c1e01fd4db44c982f123619636b74e, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T16:41:52,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/3ba55ca3ec1d4c9680805bc634df958e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/3ba55ca3ec1d4c9680805bc634df958e 2024-11-24T16:41:52,368 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/3ba55ca3ec1d4c9680805bc634df958e, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T16:41:52,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/ac3020527b184cd5b67ab35e6b0a99a9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ac3020527b184cd5b67ab35e6b0a99a9 2024-11-24T16:41:52,372 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ac3020527b184cd5b67ab35e6b0a99a9, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T16:41:52,373 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 200e097340e1bd260123b073b009f0f9 in 1267ms, sequenceid=501, compaction requested=false 2024-11-24T16:41:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:52,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-24T16:41:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-24T16:41:52,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-24T16:41:52,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1870 sec 2024-11-24T16:41:52,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.1930 sec 2024-11-24T16:41:52,406 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-24T16:41:52,407 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,407 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,407 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,407 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-24T16:41:52,407 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,407 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,407 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-24T16:41:52,407 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,407 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,407 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,408 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,408 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,409 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-24T16:41:52,409 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,409 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,409 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,409 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-24T16:41:52,409 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,409 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,409 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:52,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:41:52,555 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-24T16:41:52,555 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-24T16:41:52,555 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:52,555 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,555 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,555 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-24T16:41:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:52,555 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,555 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:52,555 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:52,555 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:52,555 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:52,555 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:52,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:52,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/080998513b484fb1b8fe1af14e4acc7f is 50, key is test_row_0/A:col10/1732466512554/Put/seqid=0 2024-11-24T16:41:52,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742171_1347 (size=12301) 2024-11-24T16:41:52,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/080998513b484fb1b8fe1af14e4acc7f 2024-11-24T16:41:52,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/f717f7557dec41f8a3bf338b72ad1a81 is 50, key is test_row_0/B:col10/1732466512554/Put/seqid=0 2024-11-24T16:41:52,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742172_1348 (size=12301) 2024-11-24T16:41:53,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/f717f7557dec41f8a3bf338b72ad1a81 2024-11-24T16:41:53,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/85ae9492a8944c3799f05f5e6fbf6f66 is 50, key is test_row_0/C:col10/1732466512554/Put/seqid=0 2024-11-24T16:41:53,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742173_1349 (size=12301) 2024-11-24T16:41:53,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/85ae9492a8944c3799f05f5e6fbf6f66 2024-11-24T16:41:53,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/080998513b484fb1b8fe1af14e4acc7f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/080998513b484fb1b8fe1af14e4acc7f 2024-11-24T16:41:53,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/080998513b484fb1b8fe1af14e4acc7f, entries=150, sequenceid=519, filesize=12.0 K 2024-11-24T16:41:53,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/f717f7557dec41f8a3bf338b72ad1a81 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f717f7557dec41f8a3bf338b72ad1a81 2024-11-24T16:41:53,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f717f7557dec41f8a3bf338b72ad1a81, entries=150, sequenceid=519, filesize=12.0 K 2024-11-24T16:41:53,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/85ae9492a8944c3799f05f5e6fbf6f66 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/85ae9492a8944c3799f05f5e6fbf6f66 2024-11-24T16:41:53,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/85ae9492a8944c3799f05f5e6fbf6f66, entries=150, sequenceid=519, filesize=12.0 K 2024-11-24T16:41:53,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 200e097340e1bd260123b073b009f0f9 in 1245ms, sequenceid=519, compaction requested=true 2024-11-24T16:41:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:41:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:41:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:53,800 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 200e097340e1bd260123b073b009f0f9:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:41:53,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:53,800 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:53,801 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:53,801 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:53,801 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/B is initiating minor compaction (all files) 2024-11-24T16:41:53,801 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/A is initiating minor compaction (all files) 2024-11-24T16:41:53,801 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/B in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:53,801 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/A in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:53,801 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d3940bfca46a49268f2541235f3a4a63, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/3ba55ca3ec1d4c9680805bc634df958e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f717f7557dec41f8a3bf338b72ad1a81] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=37.1 K 2024-11-24T16:41:53,801 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0d2cb612b38f4f58ba4fd5f3c2e209e0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5c1e01fd4db44c982f123619636b74e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/080998513b484fb1b8fe1af14e4acc7f] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=37.1 K 2024-11-24T16:41:53,801 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d3940bfca46a49268f2541235f3a4a63, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466510206 2024-11-24T16:41:53,801 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2cb612b38f4f58ba4fd5f3c2e209e0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466510206 2024-11-24T16:41:53,802 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba55ca3ec1d4c9680805bc634df958e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732466510243 2024-11-24T16:41:53,802 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c1e01fd4db44c982f123619636b74e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732466510243 2024-11-24T16:41:53,802 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f717f7557dec41f8a3bf338b72ad1a81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732466512554 2024-11-24T16:41:53,802 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 080998513b484fb1b8fe1af14e4acc7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732466512554 2024-11-24T16:41:53,808 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#A#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:53,808 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:53,808 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/505756f9cf9d450e8281488aac185386 is 50, key is test_row_0/A:col10/1732466512554/Put/seqid=0 2024-11-24T16:41:53,808 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/119789d5e5cd4dfa8d7ee5378e7deffb is 50, key is test_row_0/B:col10/1732466512554/Put/seqid=0 2024-11-24T16:41:53,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742174_1350 (size=13493) 2024-11-24T16:41:53,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742175_1351 (size=13493) 2024-11-24T16:41:54,216 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/505756f9cf9d450e8281488aac185386 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/505756f9cf9d450e8281488aac185386 2024-11-24T16:41:54,216 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/119789d5e5cd4dfa8d7ee5378e7deffb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/119789d5e5cd4dfa8d7ee5378e7deffb 2024-11-24T16:41:54,220 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/A of 200e097340e1bd260123b073b009f0f9 into 505756f9cf9d450e8281488aac185386(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:54,220 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/B of 200e097340e1bd260123b073b009f0f9 into 119789d5e5cd4dfa8d7ee5378e7deffb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:54,220 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:54,220 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:54,220 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/A, priority=13, startTime=1732466513800; duration=0sec 2024-11-24T16:41:54,220 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/B, priority=13, startTime=1732466513800; duration=0sec 2024-11-24T16:41:54,220 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:54,220 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:B 2024-11-24T16:41:54,220 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:41:54,220 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:A 2024-11-24T16:41:54,220 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:41:54,221 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:41:54,221 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 200e097340e1bd260123b073b009f0f9/C is initiating minor compaction (all files) 2024-11-24T16:41:54,221 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 200e097340e1bd260123b073b009f0f9/C in TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:54,221 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d74db40055fd4cb4b2c2b677c43c1504, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ac3020527b184cd5b67ab35e6b0a99a9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/85ae9492a8944c3799f05f5e6fbf6f66] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp, totalSize=37.1 K 2024-11-24T16:41:54,221 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d74db40055fd4cb4b2c2b677c43c1504, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732466510206 2024-11-24T16:41:54,222 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3020527b184cd5b67ab35e6b0a99a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732466510243 2024-11-24T16:41:54,222 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ae9492a8944c3799f05f5e6fbf6f66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1732466512554 2024-11-24T16:41:54,227 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 200e097340e1bd260123b073b009f0f9#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:41:54,227 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/14ba26099c4846bfbb674fc50d930bd0 is 50, key is test_row_0/C:col10/1732466512554/Put/seqid=0 2024-11-24T16:41:54,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742176_1352 (size=13459) 2024-11-24T16:41:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T16:41:54,354 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:54,635 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/14ba26099c4846bfbb674fc50d930bd0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/14ba26099c4846bfbb674fc50d930bd0 2024-11-24T16:41:54,640 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 200e097340e1bd260123b073b009f0f9/C of 200e097340e1bd260123b073b009f0f9 into 14ba26099c4846bfbb674fc50d930bd0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:41:54,640 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 200e097340e1bd260123b073b009f0f9: 2024-11-24T16:41:54,640 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., storeName=200e097340e1bd260123b073b009f0f9/C, priority=13, startTime=1732466513800; duration=0sec 2024-11-24T16:41:54,640 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:41:54,640 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 200e097340e1bd260123b073b009f0f9:C 2024-11-24T16:41:54,785 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-24T16:41:54,785 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:54,785 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:54,785 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:54,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:54,824 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-24T16:41:54,824 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:41:54,824 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:54,825 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:54,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2730 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2680 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2618 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2723 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2691 2024-11-24T16:41:54,825 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T16:41:54,825 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:41:54,825 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:41:54,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:54,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:54,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:41:54,826 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T16:41:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T16:41:54,829 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466514828"}]},"ts":"1732466514828"} 2024-11-24T16:41:54,830 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T16:41:54,830 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T16:41:54,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:41:54,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, UNASSIGN}] 2024-11-24T16:41:54,832 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, UNASSIGN 2024-11-24T16:41:54,833 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=200e097340e1bd260123b073b009f0f9, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:54,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, UNASSIGN because future has completed 2024-11-24T16:41:54,834 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:41:54,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T16:41:54,986 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:54,986 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:41:54,986 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 200e097340e1bd260123b073b009f0f9, disabling compactions & flushes 2024-11-24T16:41:54,986 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:54,986 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:54,986 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. after waiting 0 ms 2024-11-24T16:41:54,986 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:54,986 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 200e097340e1bd260123b073b009f0f9 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-24T16:41:54,986 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=A 2024-11-24T16:41:54,986 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:54,987 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=B 2024-11-24T16:41:54,987 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:54,987 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 200e097340e1bd260123b073b009f0f9, store=C 2024-11-24T16:41:54,987 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:41:54,990 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/345c434395634505828ebd36b6310a26 is 50, key is test_row_0/A:col10/1732466514784/Put/seqid=0 2024-11-24T16:41:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742177_1353 (size=9857) 2024-11-24T16:41:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T16:41:55,394 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/345c434395634505828ebd36b6310a26 2024-11-24T16:41:55,402 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/6342687f3fea4475842e1399a5cd258c is 50, key is test_row_0/B:col10/1732466514784/Put/seqid=0 2024-11-24T16:41:55,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742178_1354 (size=9857) 2024-11-24T16:41:55,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T16:41:55,806 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/6342687f3fea4475842e1399a5cd258c 2024-11-24T16:41:55,812 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/5a4c1451960441119c6b0633af123b3f is 50, key is test_row_0/C:col10/1732466514784/Put/seqid=0 2024-11-24T16:41:55,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742179_1355 (size=9857) 2024-11-24T16:41:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T16:41:56,216 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/5a4c1451960441119c6b0633af123b3f 2024-11-24T16:41:56,220 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/A/345c434395634505828ebd36b6310a26 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/345c434395634505828ebd36b6310a26 2024-11-24T16:41:56,224 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/345c434395634505828ebd36b6310a26, entries=100, sequenceid=527, filesize=9.6 K 2024-11-24T16:41:56,224 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/B/6342687f3fea4475842e1399a5cd258c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6342687f3fea4475842e1399a5cd258c 2024-11-24T16:41:56,227 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6342687f3fea4475842e1399a5cd258c, entries=100, sequenceid=527, filesize=9.6 K 2024-11-24T16:41:56,228 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/.tmp/C/5a4c1451960441119c6b0633af123b3f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/5a4c1451960441119c6b0633af123b3f 2024-11-24T16:41:56,230 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/5a4c1451960441119c6b0633af123b3f, entries=100, sequenceid=527, filesize=9.6 K 2024-11-24T16:41:56,231 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 200e097340e1bd260123b073b009f0f9 in 1245ms, sequenceid=527, compaction requested=false 2024-11-24T16:41:56,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/61f94a362ff442af8ca669a09994e7b0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/64366a0f071a42e09d73765403a79f8d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1c88afbdf72a4709b66b9f5bca61bdc0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/7233780298304ba8ab8d167b8710e8ac, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/cb418da03a574f4a942668aa0832494c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/5018dcd3cde446eb90c3dcf07f808f98, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/8631f8bd5e484ad48bc631ac495f4e52, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/abfab1dd8e4d400fa58879a09a75ec5e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/74bd58d6b5e041aaa2065bbdd2af02a7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5a05e122cf048558e4e93c1a16099c2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/65e71da7408f4a9c94dcdacace481d2b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6bfb8117277740b3bff09881b8bfef47, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/b14d77dd93fb4cd9ac243740b370316a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/9260c524142b4c5da3bfb72c123b3225, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/34ba6f87891945cba45f25139a20978f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/23b671b7f0124aa29573952bad5848b5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/dca368f8cc3c4608acdb8c88b568b95a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e6e8d560e9844a9aa360540d968089ad, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0f2890f47f8746f087f67be7a066904c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/2c8c9521bba94bd5b797847ba067dae9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/a3e9d6b0420648c295195166ecd53726, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0bceff5da9504f0bb713b8ea435232ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/75533a3950314b14b6f6ae35efe28ea5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/822dbbcb8d4241fb811910472285ef27, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab75d10be8d147c8a54e1c562ba3f87c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6760267e07be462191a35eb6c9a8dc6c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0e1d3e0a25484684ad68d39c948d2dee, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1f6e29dad2b44e799b22ccd4e2d987b0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/303084921e754a41a30179a7fca9510c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e4a3b011ef3d435e9f630361ee832890, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0eba0f3783804e11ae528cc09175157c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab9614de5a7a4a7c937372adff804acb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0d2cb612b38f4f58ba4fd5f3c2e209e0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5c1e01fd4db44c982f123619636b74e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/080998513b484fb1b8fe1af14e4acc7f] to archive 2024-11-24T16:41:56,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:56,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/61f94a362ff442af8ca669a09994e7b0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/61f94a362ff442af8ca669a09994e7b0 2024-11-24T16:41:56,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/64366a0f071a42e09d73765403a79f8d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/64366a0f071a42e09d73765403a79f8d 2024-11-24T16:41:56,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1c88afbdf72a4709b66b9f5bca61bdc0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1c88afbdf72a4709b66b9f5bca61bdc0 2024-11-24T16:41:56,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/7233780298304ba8ab8d167b8710e8ac to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/7233780298304ba8ab8d167b8710e8ac 2024-11-24T16:41:56,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/cb418da03a574f4a942668aa0832494c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/cb418da03a574f4a942668aa0832494c 2024-11-24T16:41:56,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/5018dcd3cde446eb90c3dcf07f808f98 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/5018dcd3cde446eb90c3dcf07f808f98 2024-11-24T16:41:56,239 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/8631f8bd5e484ad48bc631ac495f4e52 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/8631f8bd5e484ad48bc631ac495f4e52 2024-11-24T16:41:56,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/abfab1dd8e4d400fa58879a09a75ec5e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/abfab1dd8e4d400fa58879a09a75ec5e 2024-11-24T16:41:56,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/74bd58d6b5e041aaa2065bbdd2af02a7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/74bd58d6b5e041aaa2065bbdd2af02a7 2024-11-24T16:41:56,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5a05e122cf048558e4e93c1a16099c2 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5a05e122cf048558e4e93c1a16099c2 2024-11-24T16:41:56,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/65e71da7408f4a9c94dcdacace481d2b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/65e71da7408f4a9c94dcdacace481d2b 2024-11-24T16:41:56,243 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6bfb8117277740b3bff09881b8bfef47 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6bfb8117277740b3bff09881b8bfef47 2024-11-24T16:41:56,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/b14d77dd93fb4cd9ac243740b370316a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/b14d77dd93fb4cd9ac243740b370316a 2024-11-24T16:41:56,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/9260c524142b4c5da3bfb72c123b3225 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/9260c524142b4c5da3bfb72c123b3225 2024-11-24T16:41:56,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/34ba6f87891945cba45f25139a20978f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/34ba6f87891945cba45f25139a20978f 2024-11-24T16:41:56,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/23b671b7f0124aa29573952bad5848b5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/23b671b7f0124aa29573952bad5848b5 2024-11-24T16:41:56,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/dca368f8cc3c4608acdb8c88b568b95a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/dca368f8cc3c4608acdb8c88b568b95a 2024-11-24T16:41:56,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e6e8d560e9844a9aa360540d968089ad to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e6e8d560e9844a9aa360540d968089ad 2024-11-24T16:41:56,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0f2890f47f8746f087f67be7a066904c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0f2890f47f8746f087f67be7a066904c 2024-11-24T16:41:56,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/2c8c9521bba94bd5b797847ba067dae9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/2c8c9521bba94bd5b797847ba067dae9 2024-11-24T16:41:56,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/a3e9d6b0420648c295195166ecd53726 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/a3e9d6b0420648c295195166ecd53726 2024-11-24T16:41:56,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0bceff5da9504f0bb713b8ea435232ea to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0bceff5da9504f0bb713b8ea435232ea 2024-11-24T16:41:56,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/75533a3950314b14b6f6ae35efe28ea5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/75533a3950314b14b6f6ae35efe28ea5 2024-11-24T16:41:56,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/822dbbcb8d4241fb811910472285ef27 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/822dbbcb8d4241fb811910472285ef27 2024-11-24T16:41:56,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab75d10be8d147c8a54e1c562ba3f87c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab75d10be8d147c8a54e1c562ba3f87c 2024-11-24T16:41:56,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6760267e07be462191a35eb6c9a8dc6c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/6760267e07be462191a35eb6c9a8dc6c 2024-11-24T16:41:56,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0e1d3e0a25484684ad68d39c948d2dee to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0e1d3e0a25484684ad68d39c948d2dee 2024-11-24T16:41:56,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1f6e29dad2b44e799b22ccd4e2d987b0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/1f6e29dad2b44e799b22ccd4e2d987b0 2024-11-24T16:41:56,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/303084921e754a41a30179a7fca9510c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/303084921e754a41a30179a7fca9510c 2024-11-24T16:41:56,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e4a3b011ef3d435e9f630361ee832890 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/e4a3b011ef3d435e9f630361ee832890 2024-11-24T16:41:56,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0eba0f3783804e11ae528cc09175157c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0eba0f3783804e11ae528cc09175157c 2024-11-24T16:41:56,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab9614de5a7a4a7c937372adff804acb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/ab9614de5a7a4a7c937372adff804acb 2024-11-24T16:41:56,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0d2cb612b38f4f58ba4fd5f3c2e209e0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/0d2cb612b38f4f58ba4fd5f3c2e209e0 2024-11-24T16:41:56,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5c1e01fd4db44c982f123619636b74e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/f5c1e01fd4db44c982f123619636b74e 2024-11-24T16:41:56,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/080998513b484fb1b8fe1af14e4acc7f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/080998513b484fb1b8fe1af14e4acc7f 2024-11-24T16:41:56,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f357cc5fff2d4bc4ad7cd0cf208ea313, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/73412c5b86654edb8858f04e5f6efbc2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/cfa12b5f4d3c4a4fb631650626a00ca1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1e8539507bfa4fa68155ed1af80d854c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4c542dc9adaf43a284168bf63f4d7593, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/2e3bd9b2546446449378ea00a9601cbf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/5d77211a3b634000bfe4c335df711030, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/63a08e1fc10b4b54be2f6fd185fec924, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/181a65ca9d3d47ca96e9118c1e0399fb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e4ceda7db84b4ee58a79332d1c0082f6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4f8cc86366d84b049d4e6f19c9912d36, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/ccc21e441b2e410db4508181110d190d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/fa8a5b1bfaf74547ac3a7ce18109a130, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/074d65caf6aa414dbc45c58728c0d170, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/a9e95d112aac4e9f8d0ee433f6183cec, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/baabec57acae483b939aadcf8df7d269, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1187868c5ed8470e8a7c7503c340a0b1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c021e5d7b5c145e2b07b16a81a52e9ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/70e9a19f6cad46cbbf36c9a1e9558bef, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e47903d89c8a4db3b5ba4996e02e99d0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/92546cadf3d348469adf56a42dffac79, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e2721d30d1d34b589d3db698806c5bb1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c0f2d9c2725f483f91832d4f40a9efb7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c3664720067d46d1b68c35978a0f504b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/422e96fbb48a4b15a2200ad356eb3055, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6a8bad7eb148416f8a7ec565f2608913, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/15498381706b4dbdb3d7c0a7867271c8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/270f8145b58d4c7fab78c55b77fa81d2, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/0b12db530d3a482eb925d49c04fdaca9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/898f5b2eeb7f48e492d8a46b4e17653e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d3940bfca46a49268f2541235f3a4a63, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d83da353eb1d4b52b1849325366c8b24, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/3ba55ca3ec1d4c9680805bc634df958e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f717f7557dec41f8a3bf338b72ad1a81] to archive 2024-11-24T16:41:56,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:56,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f357cc5fff2d4bc4ad7cd0cf208ea313 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f357cc5fff2d4bc4ad7cd0cf208ea313 2024-11-24T16:41:56,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/73412c5b86654edb8858f04e5f6efbc2 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/73412c5b86654edb8858f04e5f6efbc2 2024-11-24T16:41:56,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/cfa12b5f4d3c4a4fb631650626a00ca1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/cfa12b5f4d3c4a4fb631650626a00ca1 2024-11-24T16:41:56,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1e8539507bfa4fa68155ed1af80d854c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1e8539507bfa4fa68155ed1af80d854c 2024-11-24T16:41:56,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4c542dc9adaf43a284168bf63f4d7593 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4c542dc9adaf43a284168bf63f4d7593 2024-11-24T16:41:56,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/2e3bd9b2546446449378ea00a9601cbf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/2e3bd9b2546446449378ea00a9601cbf 2024-11-24T16:41:56,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/b76e6ea1931d44c7bd7ab43ab3f9c2ac 2024-11-24T16:41:56,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/5d77211a3b634000bfe4c335df711030 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/5d77211a3b634000bfe4c335df711030 2024-11-24T16:41:56,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/63a08e1fc10b4b54be2f6fd185fec924 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/63a08e1fc10b4b54be2f6fd185fec924 2024-11-24T16:41:56,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/181a65ca9d3d47ca96e9118c1e0399fb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/181a65ca9d3d47ca96e9118c1e0399fb 2024-11-24T16:41:56,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e4ceda7db84b4ee58a79332d1c0082f6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e4ceda7db84b4ee58a79332d1c0082f6 2024-11-24T16:41:56,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4f8cc86366d84b049d4e6f19c9912d36 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/4f8cc86366d84b049d4e6f19c9912d36 2024-11-24T16:41:56,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/ccc21e441b2e410db4508181110d190d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/ccc21e441b2e410db4508181110d190d 2024-11-24T16:41:56,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/fa8a5b1bfaf74547ac3a7ce18109a130 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/fa8a5b1bfaf74547ac3a7ce18109a130 2024-11-24T16:41:56,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/074d65caf6aa414dbc45c58728c0d170 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/074d65caf6aa414dbc45c58728c0d170 2024-11-24T16:41:56,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/a9e95d112aac4e9f8d0ee433f6183cec to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/a9e95d112aac4e9f8d0ee433f6183cec 2024-11-24T16:41:56,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/baabec57acae483b939aadcf8df7d269 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/baabec57acae483b939aadcf8df7d269 2024-11-24T16:41:56,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1187868c5ed8470e8a7c7503c340a0b1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/1187868c5ed8470e8a7c7503c340a0b1 2024-11-24T16:41:56,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c021e5d7b5c145e2b07b16a81a52e9ea to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c021e5d7b5c145e2b07b16a81a52e9ea 2024-11-24T16:41:56,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/70e9a19f6cad46cbbf36c9a1e9558bef to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/70e9a19f6cad46cbbf36c9a1e9558bef 2024-11-24T16:41:56,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e47903d89c8a4db3b5ba4996e02e99d0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e47903d89c8a4db3b5ba4996e02e99d0 2024-11-24T16:41:56,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/92546cadf3d348469adf56a42dffac79 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/92546cadf3d348469adf56a42dffac79 2024-11-24T16:41:56,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e2721d30d1d34b589d3db698806c5bb1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/e2721d30d1d34b589d3db698806c5bb1 2024-11-24T16:41:56,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c0f2d9c2725f483f91832d4f40a9efb7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c0f2d9c2725f483f91832d4f40a9efb7 2024-11-24T16:41:56,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c3664720067d46d1b68c35978a0f504b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/c3664720067d46d1b68c35978a0f504b 2024-11-24T16:41:56,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/422e96fbb48a4b15a2200ad356eb3055 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/422e96fbb48a4b15a2200ad356eb3055 2024-11-24T16:41:56,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6a8bad7eb148416f8a7ec565f2608913 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6a8bad7eb148416f8a7ec565f2608913 2024-11-24T16:41:56,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/15498381706b4dbdb3d7c0a7867271c8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/15498381706b4dbdb3d7c0a7867271c8 2024-11-24T16:41:56,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/270f8145b58d4c7fab78c55b77fa81d2 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/270f8145b58d4c7fab78c55b77fa81d2 2024-11-24T16:41:56,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/0b12db530d3a482eb925d49c04fdaca9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/0b12db530d3a482eb925d49c04fdaca9 2024-11-24T16:41:56,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/898f5b2eeb7f48e492d8a46b4e17653e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/898f5b2eeb7f48e492d8a46b4e17653e 2024-11-24T16:41:56,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d3940bfca46a49268f2541235f3a4a63 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d3940bfca46a49268f2541235f3a4a63 2024-11-24T16:41:56,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d83da353eb1d4b52b1849325366c8b24 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/d83da353eb1d4b52b1849325366c8b24 2024-11-24T16:41:56,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/3ba55ca3ec1d4c9680805bc634df958e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/3ba55ca3ec1d4c9680805bc634df958e 2024-11-24T16:41:56,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f717f7557dec41f8a3bf338b72ad1a81 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/f717f7557dec41f8a3bf338b72ad1a81 2024-11-24T16:41:56,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/563ba19fdb76434f97025cbb5cc296ed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9dc189d8881b4f3db072fec6470cb3db, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f913158ed0d946018af1ffe85a3d8075, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/586d0973cbe84d96a3d9116dee1e54c3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/99cc27e4cc834dffb4205ba5d3089879, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f0df926ccf8e4bdf8419123eeda10f46, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7e57d00ea02447d3a1c17dca0dd6b936, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b55e11d6aed14bedbcd4938a2116b2b3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/12d0877db0874ee8a2aab8fdeb3b8e2d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d9fd466ee83a4d55ad1c30d0b73d8bbe, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/0b38d75c52a744fdb8669855884748e3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ccfad34dc5084ad1a23b8348baef16a7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a0def2a311244e4eb702625ea87ee4aa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9024168278aa4bc280893fccc9370403, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b7daf792e26e489697d8420252d79f12, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b25c3dcd914944449f4f917cb64a8a52, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/24ff71f5fd92477c86899b69d1f0ddee, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ec29e306421e4367ace8905e68a835c9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cc43561a5f9a40299bceb8efabd2f6b7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/c634e077350843e1b01d4d91c6f051ac, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/62f2ca9df42e412db770f0b5bf38b159, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/581681c638d84f33bea77b70a53cfbf8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7f02fd8e0d8943fca4ef9f0567760b80, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3bc255eaf8a2486c887ee4fa4131677d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cd6a1ec313b04cd6a0e625675ddafebc, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/6e387106620d401d854d18df4ca53674, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3ece7223e8444516aac1ab2bcf6e2e76, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/fa2f9830e90a418eb2b0e5fdfe2d3b67, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/56976f3f6c3a467db6567951f6122397, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a024431c2eec482281775138771e1688, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d74db40055fd4cb4b2c2b677c43c1504, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a97a21bef74c4d3b811bbf75cff85089, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ac3020527b184cd5b67ab35e6b0a99a9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/85ae9492a8944c3799f05f5e6fbf6f66] to archive 2024-11-24T16:41:56,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:41:56,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/563ba19fdb76434f97025cbb5cc296ed to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/563ba19fdb76434f97025cbb5cc296ed 2024-11-24T16:41:56,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9dc189d8881b4f3db072fec6470cb3db to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9dc189d8881b4f3db072fec6470cb3db 2024-11-24T16:41:56,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f913158ed0d946018af1ffe85a3d8075 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f913158ed0d946018af1ffe85a3d8075 2024-11-24T16:41:56,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/586d0973cbe84d96a3d9116dee1e54c3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/586d0973cbe84d96a3d9116dee1e54c3 2024-11-24T16:41:56,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/99cc27e4cc834dffb4205ba5d3089879 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/99cc27e4cc834dffb4205ba5d3089879 2024-11-24T16:41:56,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f0df926ccf8e4bdf8419123eeda10f46 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/f0df926ccf8e4bdf8419123eeda10f46 2024-11-24T16:41:56,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7e57d00ea02447d3a1c17dca0dd6b936 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7e57d00ea02447d3a1c17dca0dd6b936 2024-11-24T16:41:56,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b55e11d6aed14bedbcd4938a2116b2b3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b55e11d6aed14bedbcd4938a2116b2b3 2024-11-24T16:41:56,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/12d0877db0874ee8a2aab8fdeb3b8e2d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/12d0877db0874ee8a2aab8fdeb3b8e2d 2024-11-24T16:41:56,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d9fd466ee83a4d55ad1c30d0b73d8bbe to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d9fd466ee83a4d55ad1c30d0b73d8bbe 2024-11-24T16:41:56,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/0b38d75c52a744fdb8669855884748e3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/0b38d75c52a744fdb8669855884748e3 2024-11-24T16:41:56,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ccfad34dc5084ad1a23b8348baef16a7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ccfad34dc5084ad1a23b8348baef16a7 2024-11-24T16:41:56,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a0def2a311244e4eb702625ea87ee4aa to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a0def2a311244e4eb702625ea87ee4aa 2024-11-24T16:41:56,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9024168278aa4bc280893fccc9370403 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/9024168278aa4bc280893fccc9370403 2024-11-24T16:41:56,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b7daf792e26e489697d8420252d79f12 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b7daf792e26e489697d8420252d79f12 2024-11-24T16:41:56,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b25c3dcd914944449f4f917cb64a8a52 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/b25c3dcd914944449f4f917cb64a8a52 2024-11-24T16:41:56,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/24ff71f5fd92477c86899b69d1f0ddee to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/24ff71f5fd92477c86899b69d1f0ddee 2024-11-24T16:41:56,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ec29e306421e4367ace8905e68a835c9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ec29e306421e4367ace8905e68a835c9 2024-11-24T16:41:56,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cc43561a5f9a40299bceb8efabd2f6b7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cc43561a5f9a40299bceb8efabd2f6b7 2024-11-24T16:41:56,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/c634e077350843e1b01d4d91c6f051ac to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/c634e077350843e1b01d4d91c6f051ac 2024-11-24T16:41:56,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/62f2ca9df42e412db770f0b5bf38b159 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/62f2ca9df42e412db770f0b5bf38b159 2024-11-24T16:41:56,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/581681c638d84f33bea77b70a53cfbf8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/581681c638d84f33bea77b70a53cfbf8 2024-11-24T16:41:56,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7f02fd8e0d8943fca4ef9f0567760b80 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/7f02fd8e0d8943fca4ef9f0567760b80 2024-11-24T16:41:56,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3bc255eaf8a2486c887ee4fa4131677d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3bc255eaf8a2486c887ee4fa4131677d 2024-11-24T16:41:56,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cd6a1ec313b04cd6a0e625675ddafebc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/cd6a1ec313b04cd6a0e625675ddafebc 2024-11-24T16:41:56,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/6e387106620d401d854d18df4ca53674 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/6e387106620d401d854d18df4ca53674 2024-11-24T16:41:56,362 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3ece7223e8444516aac1ab2bcf6e2e76 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/3ece7223e8444516aac1ab2bcf6e2e76 2024-11-24T16:41:56,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/fa2f9830e90a418eb2b0e5fdfe2d3b67 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/fa2f9830e90a418eb2b0e5fdfe2d3b67 2024-11-24T16:41:56,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/56976f3f6c3a467db6567951f6122397 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/56976f3f6c3a467db6567951f6122397 2024-11-24T16:41:56,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a024431c2eec482281775138771e1688 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a024431c2eec482281775138771e1688 2024-11-24T16:41:56,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d74db40055fd4cb4b2c2b677c43c1504 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/d74db40055fd4cb4b2c2b677c43c1504 2024-11-24T16:41:56,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a97a21bef74c4d3b811bbf75cff85089 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/a97a21bef74c4d3b811bbf75cff85089 2024-11-24T16:41:56,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ac3020527b184cd5b67ab35e6b0a99a9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/ac3020527b184cd5b67ab35e6b0a99a9 2024-11-24T16:41:56,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/85ae9492a8944c3799f05f5e6fbf6f66 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/85ae9492a8944c3799f05f5e6fbf6f66 2024-11-24T16:41:56,372 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/recovered.edits/530.seqid, newMaxSeqId=530, maxSeqId=1 2024-11-24T16:41:56,373 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. 2024-11-24T16:41:56,373 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 200e097340e1bd260123b073b009f0f9: Waiting for close lock at 1732466514986Running coprocessor pre-close hooks at 1732466514986Disabling compacts and flushes for region at 1732466514986Disabling writes for close at 1732466514986Obtaining lock to block concurrent updates at 1732466514986Preparing flush snapshotting stores in 200e097340e1bd260123b073b009f0f9 at 1732466514986Finished memstore snapshotting TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732466514987 (+1 ms)Flushing stores of TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9. at 1732466514987Flushing 200e097340e1bd260123b073b009f0f9/A: creating writer at 1732466514987Flushing 200e097340e1bd260123b073b009f0f9/A: appending metadata at 1732466514990 (+3 ms)Flushing 200e097340e1bd260123b073b009f0f9/A: closing flushed file at 1732466514990Flushing 200e097340e1bd260123b073b009f0f9/B: creating writer at 1732466515399 (+409 ms)Flushing 200e097340e1bd260123b073b009f0f9/B: appending metadata at 1732466515401 (+2 ms)Flushing 200e097340e1bd260123b073b009f0f9/B: closing flushed file at 1732466515402 (+1 ms)Flushing 200e097340e1bd260123b073b009f0f9/C: creating writer at 1732466515810 (+408 ms)Flushing 200e097340e1bd260123b073b009f0f9/C: appending metadata at 1732466515812 (+2 ms)Flushing 200e097340e1bd260123b073b009f0f9/C: closing flushed file at 1732466515812Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@484f3a3a: reopening flushed file at 1732466516219 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5286629: reopening flushed file at 1732466516224 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@338d65f7: reopening flushed file at 1732466516227 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 200e097340e1bd260123b073b009f0f9 in 1245ms, sequenceid=527, compaction requested=false at 1732466516231 (+4 ms)Writing region close event to WAL at 1732466516370 (+139 ms)Running coprocessor post-close hooks at 1732466516373 (+3 ms)Closed at 1732466516373 2024-11-24T16:41:56,375 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:56,375 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=200e097340e1bd260123b073b009f0f9, regionState=CLOSED 2024-11-24T16:41:56,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:56,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-24T16:41:56,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 200e097340e1bd260123b073b009f0f9, server=e335232f7491,33607,1732466431346 in 1.5430 sec 2024-11-24T16:41:56,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-24T16:41:56,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=200e097340e1bd260123b073b009f0f9, UNASSIGN in 1.5470 sec 2024-11-24T16:41:56,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-24T16:41:56,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5500 sec 2024-11-24T16:41:56,383 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466516382"}]},"ts":"1732466516382"} 2024-11-24T16:41:56,384 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T16:41:56,384 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T16:41:56,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5580 sec 2024-11-24T16:41:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T16:41:56,974 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:56,975 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T16:41:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:56,976 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T16:41:56,977 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:56,980 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:56,981 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/recovered.edits] 2024-11-24T16:41:56,984 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/345c434395634505828ebd36b6310a26 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/345c434395634505828ebd36b6310a26 2024-11-24T16:41:56,984 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/505756f9cf9d450e8281488aac185386 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/A/505756f9cf9d450e8281488aac185386 2024-11-24T16:41:56,986 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/119789d5e5cd4dfa8d7ee5378e7deffb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/119789d5e5cd4dfa8d7ee5378e7deffb 2024-11-24T16:41:56,987 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6342687f3fea4475842e1399a5cd258c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/B/6342687f3fea4475842e1399a5cd258c 2024-11-24T16:41:56,989 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/14ba26099c4846bfbb674fc50d930bd0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/14ba26099c4846bfbb674fc50d930bd0 2024-11-24T16:41:56,990 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/5a4c1451960441119c6b0633af123b3f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/C/5a4c1451960441119c6b0633af123b3f 2024-11-24T16:41:56,992 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/recovered.edits/530.seqid to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9/recovered.edits/530.seqid 2024-11-24T16:41:56,993 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/200e097340e1bd260123b073b009f0f9 2024-11-24T16:41:56,993 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T16:41:56,995 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:56,997 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T16:41:56,999 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T16:41:57,000 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:57,000 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T16:41:57,000 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732466517000"}]},"ts":"9223372036854775807"} 2024-11-24T16:41:57,002 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T16:41:57,002 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 200e097340e1bd260123b073b009f0f9, NAME => 'TestAcidGuarantees,,1732466490192.200e097340e1bd260123b073b009f0f9.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T16:41:57,002 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T16:41:57,002 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732466517002"}]},"ts":"9223372036854775807"} 2024-11-24T16:41:57,003 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T16:41:57,004 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:57,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-24T16:41:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T16:41:57,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T16:41:57,084 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:57,093 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 249), OpenFileDescriptor=457 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=474 (was 503), ProcessCount=11 (was 11), AvailableMemoryMB=6388 (was 6736) 2024-11-24T16:41:57,102 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=474, ProcessCount=11, AvailableMemoryMB=6387 2024-11-24T16:41:57,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:41:57,103 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:41:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:57,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:41:57,105 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:57,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-24T16:41:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T16:41:57,106 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:41:57,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742180_1356 (size=1061) 2024-11-24T16:41:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T16:41:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T16:41:57,512 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:41:57,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742181_1357 (size=53) 2024-11-24T16:41:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T16:41:57,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:41:57,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3abd8569b30f1874fc795b3a06786690, disabling compactions & flushes 2024-11-24T16:41:57,918 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:57,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:57,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. after waiting 0 ms 2024-11-24T16:41:57,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:57,918 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:57,918 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3abd8569b30f1874fc795b3a06786690: Waiting for close lock at 1732466517918Disabling compacts and flushes for region at 1732466517918Disabling writes for close at 1732466517918Writing region close event to WAL at 1732466517918Closed at 1732466517918 2024-11-24T16:41:57,919 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:41:57,919 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732466517919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466517919"}]},"ts":"1732466517919"} 2024-11-24T16:41:57,921 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:41:57,921 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:41:57,921 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466517921"}]},"ts":"1732466517921"} 2024-11-24T16:41:57,922 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T16:41:57,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, ASSIGN}] 2024-11-24T16:41:57,924 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, ASSIGN 2024-11-24T16:41:57,924 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, ASSIGN; state=OFFLINE, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=false 2024-11-24T16:41:58,075 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:58,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, ASSIGN because future has completed 2024-11-24T16:41:58,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:58,231 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:58,232 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:41:58,232 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,232 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:41:58,232 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,232 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,233 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,234 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:58,235 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3abd8569b30f1874fc795b3a06786690 columnFamilyName A 2024-11-24T16:41:58,235 DEBUG [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:58,235 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(327): Store=3abd8569b30f1874fc795b3a06786690/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:58,235 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,236 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:58,236 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3abd8569b30f1874fc795b3a06786690 columnFamilyName B 2024-11-24T16:41:58,236 DEBUG [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:58,236 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(327): Store=3abd8569b30f1874fc795b3a06786690/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:58,237 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,237 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:41:58,237 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3abd8569b30f1874fc795b3a06786690 columnFamilyName C 2024-11-24T16:41:58,237 DEBUG [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:41:58,238 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(327): Store=3abd8569b30f1874fc795b3a06786690/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:41:58,238 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:58,238 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,239 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,239 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,239 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,239 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,240 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:41:58,241 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,242 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:41:58,243 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 3abd8569b30f1874fc795b3a06786690; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71970373, jitterRate=0.07244212925434113}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:41:58,243 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:58,243 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 3abd8569b30f1874fc795b3a06786690: Running coprocessor pre-open hook at 1732466518232Writing region info on filesystem at 1732466518232Initializing all the Stores at 1732466518233 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466518233Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466518233Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466518233Cleaning up temporary data from old regions at 1732466518239 (+6 ms)Running coprocessor post-open hooks at 1732466518243 (+4 ms)Region opened successfully at 1732466518243 2024-11-24T16:41:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T16:41:58,244 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., pid=95, masterSystemTime=1732466518228 2024-11-24T16:41:58,246 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:58,246 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:58,246 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:58,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:58,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-24T16:41:58,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 in 171 msec 2024-11-24T16:41:58,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-24T16:41:58,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, ASSIGN in 326 msec 2024-11-24T16:41:58,251 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:41:58,251 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466518251"}]},"ts":"1732466518251"} 2024-11-24T16:41:58,252 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T16:41:58,253 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:41:58,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-24T16:41:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T16:41:59,254 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:41:59,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ab9835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:59,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:41:59,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:41:59,256 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:41:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:41:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:41:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b8b5c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:41:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:41:59,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:41:59,257 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:41:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68795bd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:41:59,258 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:41:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:41:59,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:41:59,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:41:59,260 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:41:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,37979,1732466430336 2024-11-24T16:41:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f0b40dd 2024-11-24T16:41:59,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:41:59,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:41:59,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:41:59,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:41:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T16:41:59,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742182_1358 (size=1097) 2024-11-24T16:41:59,335 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T16:41:59,674 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T16:41:59,674 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T16:41:59,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:41:59,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, REOPEN/MOVE}] 2024-11-24T16:41:59,679 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, REOPEN/MOVE 2024-11-24T16:41:59,679 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:59,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, REOPEN/MOVE because future has completed 2024-11-24T16:41:59,681 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:41:59,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:41:59,833 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:59,833 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:41:59,833 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 3abd8569b30f1874fc795b3a06786690, disabling compactions & flushes 2024-11-24T16:41:59,833 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:59,833 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:59,833 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. after waiting 0 ms 2024-11-24T16:41:59,833 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:59,836 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T16:41:59,837 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:41:59,837 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 3abd8569b30f1874fc795b3a06786690: Waiting for close lock at 1732466519833Running coprocessor pre-close hooks at 1732466519833Disabling compacts and flushes for region at 1732466519833Disabling writes for close at 1732466519833Writing region close event to WAL at 1732466519833Running coprocessor post-close hooks at 1732466519837 (+4 ms)Closed at 1732466519837 2024-11-24T16:41:59,837 WARN [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 3abd8569b30f1874fc795b3a06786690 to self. 2024-11-24T16:41:59,839 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:41:59,839 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=CLOSED 2024-11-24T16:41:59,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:41:59,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-24T16:41:59,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 in 160 msec 2024-11-24T16:41:59,843 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, REOPEN/MOVE; state=CLOSED, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=true 2024-11-24T16:41:59,993 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:41:59,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, REOPEN/MOVE because future has completed 2024-11-24T16:41:59,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:42:00,149 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,149 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:42:00,150 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,150 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:42:00,150 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,150 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,151 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,152 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:42:00,152 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3abd8569b30f1874fc795b3a06786690 columnFamilyName A 2024-11-24T16:42:00,153 DEBUG [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:00,153 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(327): Store=3abd8569b30f1874fc795b3a06786690/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:42:00,154 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,154 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:42:00,154 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3abd8569b30f1874fc795b3a06786690 columnFamilyName B 2024-11-24T16:42:00,154 DEBUG [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:00,155 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(327): Store=3abd8569b30f1874fc795b3a06786690/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:42:00,155 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,155 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:42:00,155 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3abd8569b30f1874fc795b3a06786690 columnFamilyName C 2024-11-24T16:42:00,155 DEBUG [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:00,155 INFO [StoreOpener-3abd8569b30f1874fc795b3a06786690-1 {}] regionserver.HStore(327): Store=3abd8569b30f1874fc795b3a06786690/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:42:00,156 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,156 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,156 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,157 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,158 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,158 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,158 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:42:00,159 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,160 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 3abd8569b30f1874fc795b3a06786690; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66938627, jitterRate=-0.0025367289781570435}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:42:00,160 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,160 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 3abd8569b30f1874fc795b3a06786690: Running coprocessor pre-open hook at 1732466520150Writing region info on filesystem at 1732466520150Initializing all the Stores at 1732466520150Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466520151 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466520151Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466520151Cleaning up temporary data from old regions at 1732466520158 (+7 ms)Running coprocessor post-open hooks at 1732466520160 (+2 ms)Region opened successfully at 1732466520160 2024-11-24T16:42:00,161 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., pid=100, masterSystemTime=1732466520146 2024-11-24T16:42:00,163 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,163 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,163 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=OPEN, openSeqNum=5, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:42:00,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-24T16:42:00,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 in 170 msec 2024-11-24T16:42:00,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-24T16:42:00,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, REOPEN/MOVE in 489 msec 2024-11-24T16:42:00,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-24T16:42:00,170 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-24T16:42:00,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 908 msec 2024-11-24T16:42:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-24T16:42:00,173 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:00,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11317fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573e025b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,176 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71c3c48b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,177 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a334106, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,178 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17c7df9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,180 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791455c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aec1645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,183 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e15587, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,184 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c20ce47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,185 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@262581b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,186 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b93d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,188 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b36c1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,189 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d3cdd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da8c649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eeae465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,193 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401bb9c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,194 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a7a6151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,196 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b46dceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,197 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b60598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:00,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:00,199 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7932e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:00,200 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7215473a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:00,201 DEBUG [Thread-1636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,201 DEBUG [Thread-1630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,201 DEBUG [Thread-1632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,201 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,202 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,202 DEBUG [Thread-1623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,203 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,205 DEBUG [Thread-1628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,205 DEBUG [Thread-1634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,206 DEBUG [Thread-1628 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,206 DEBUG [Thread-1628 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,207 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-24T16:42:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,209 DEBUG [Thread-1625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T16:42:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:00,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:00,211 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,216 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:00,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:42:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:00,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:00,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112456a1938df70d48f9bdbe9326b27927f2_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466520217/Put/seqid=0 2024-11-24T16:42:00,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466520252, max timestamp=1732466520276Num uniques 859; 2024-11-24T16:42:00,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:00,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466520252, max timestamp=1732466520276Num uniques 859; 2024-11-24T16:42:00,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466520252, max timestamp=1732466520276Num uniques 859; 2024-11-24T16:42:00,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:00,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:00,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742183_1359 (size=12154) 2024-11-24T16:42:00,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:00,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112456a1938df70d48f9bdbe9326b27927f2_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456a1938df70d48f9bdbe9326b27927f2_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:00,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/cca2c61bb50b4f77b85ad07f0af6d259, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:00,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/cca2c61bb50b4f77b85ad07f0af6d259 is 175, key is test_row_0/A:col10/1732466520217/Put/seqid=0 2024-11-24T16:42:00,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742184_1360 (size=30955) 2024-11-24T16:42:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T16:42:00,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466580306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466580308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466580309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466580316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466580317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:00,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:00,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466580424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466580434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466580434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466580434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466580435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T16:42:00,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466580644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466580654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466580654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466580654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466580654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:00,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:00,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,706 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/cca2c61bb50b4f77b85ad07f0af6d259 2024-11-24T16:42:00,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/a3a3dda78e724940ab0d95f97971ebcd is 50, key is test_row_0/B:col10/1732466520217/Put/seqid=0 2024-11-24T16:42:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742185_1361 (size=12001) 2024-11-24T16:42:00,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:00,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:00,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T16:42:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466580953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466580974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466580974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466580974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466580974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:00,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:00,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:00,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:00,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/a3a3dda78e724940ab0d95f97971ebcd 2024-11-24T16:42:01,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:01,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/4ec904296eab40fa8d015ea837dda863 is 50, key is test_row_0/C:col10/1732466520217/Put/seqid=0 2024-11-24T16:42:01,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742186_1362 (size=12001) 2024-11-24T16:42:01,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:01,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:01,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:01,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:01,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T16:42:01,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:01,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:01,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:01,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466581474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:01,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466581494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:01,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466581494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:01,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466581494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:01,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466581494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:01,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:01,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/4ec904296eab40fa8d015ea837dda863 2024-11-24T16:42:01,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/cca2c61bb50b4f77b85ad07f0af6d259 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/cca2c61bb50b4f77b85ad07f0af6d259 2024-11-24T16:42:01,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/cca2c61bb50b4f77b85ad07f0af6d259, entries=150, sequenceid=14, filesize=30.2 K 2024-11-24T16:42:01,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/a3a3dda78e724940ab0d95f97971ebcd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a3a3dda78e724940ab0d95f97971ebcd 2024-11-24T16:42:01,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a3a3dda78e724940ab0d95f97971ebcd, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T16:42:01,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/4ec904296eab40fa8d015ea837dda863 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/4ec904296eab40fa8d015ea837dda863 2024-11-24T16:42:01,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/4ec904296eab40fa8d015ea837dda863, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T16:42:01,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3abd8569b30f1874fc795b3a06786690 in 1358ms, sequenceid=14, compaction requested=false 2024-11-24T16:42:01,578 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T16:42:01,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:01,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T16:42:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:01,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T16:42:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:01,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248aeb96497e0d48deb4c8cd91baa8f4b0_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466520314/Put/seqid=0 2024-11-24T16:42:01,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742187_1363 (size=12154) 2024-11-24T16:42:01,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:01,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248aeb96497e0d48deb4c8cd91baa8f4b0_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248aeb96497e0d48deb4c8cd91baa8f4b0_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9f42a5145c52427d860c7cfd12b2ca92, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9f42a5145c52427d860c7cfd12b2ca92 is 175, key is test_row_0/A:col10/1732466520314/Put/seqid=0 2024-11-24T16:42:01,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742188_1364 (size=30955) 2024-11-24T16:42:01,899 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:42:02,022 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9f42a5145c52427d860c7cfd12b2ca92 2024-11-24T16:42:02,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/6f14784ea3c34414a79d0ff5318b5a4c is 50, key is test_row_0/B:col10/1732466520314/Put/seqid=0 2024-11-24T16:42:02,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742189_1365 (size=12001) 2024-11-24T16:42:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T16:42:02,435 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/6f14784ea3c34414a79d0ff5318b5a4c 2024-11-24T16:42:02,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/44a8d38fdb384de1838e4beba59c151f is 50, key is test_row_0/C:col10/1732466520314/Put/seqid=0 2024-11-24T16:42:02,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742190_1366 (size=12001) 2024-11-24T16:42:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:02,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:02,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466582522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466582522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466582524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466582524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466582525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,538 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 3abd8569b30f1874fc795b3a06786690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466582634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466582634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466582643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466582643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,848 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/44a8d38fdb384de1838e4beba59c151f 2024-11-24T16:42:02,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9f42a5145c52427d860c7cfd12b2ca92 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9f42a5145c52427d860c7cfd12b2ca92 2024-11-24T16:42:02,856 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9f42a5145c52427d860c7cfd12b2ca92, entries=150, sequenceid=49, filesize=30.2 K 2024-11-24T16:42:02,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/6f14784ea3c34414a79d0ff5318b5a4c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/6f14784ea3c34414a79d0ff5318b5a4c 2024-11-24T16:42:02,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466582854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466582854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466582854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466582854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:02,862 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/6f14784ea3c34414a79d0ff5318b5a4c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T16:42:02,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/44a8d38fdb384de1838e4beba59c151f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/44a8d38fdb384de1838e4beba59c151f 2024-11-24T16:42:02,866 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/44a8d38fdb384de1838e4beba59c151f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T16:42:02,866 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 3abd8569b30f1874fc795b3a06786690 in 1269ms, sequenceid=49, compaction requested=false 2024-11-24T16:42:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:02,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-24T16:42:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-24T16:42:02,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-24T16:42:02,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6520 sec 2024-11-24T16:42:02,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.6640 sec 2024-11-24T16:42:03,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:03,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:42:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:03,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245d9c7ccb36884328ba0cdcc8a4bbc978_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466522523/Put/seqid=0 2024-11-24T16:42:03,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742191_1367 (size=14594) 2024-11-24T16:42:03,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466523202, max timestamp=1732466523214Num uniques 859; 2024-11-24T16:42:03,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466523202, max timestamp=1732466523214Num uniques 859; 2024-11-24T16:42:03,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:03,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:03,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466523202, max timestamp=1732466523214Num uniques 859; 2024-11-24T16:42:03,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:03,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466583251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466583252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466583253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466583266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466583373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466583373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466583373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,384 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466583374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:03,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245d9c7ccb36884328ba0cdcc8a4bbc978_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d9c7ccb36884328ba0cdcc8a4bbc978_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:03,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/015275c0b9b64ca198d6d5233e1c0535, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:03,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/015275c0b9b64ca198d6d5233e1c0535 is 175, key is test_row_0/A:col10/1732466522523/Put/seqid=0 2024-11-24T16:42:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742192_1368 (size=39549) 2024-11-24T16:42:03,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466583593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466583593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466583593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466583593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466583904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466583904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466583904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466583913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:03,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/015275c0b9b64ca198d6d5233e1c0535 2024-11-24T16:42:03,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/86640e5b1074454fafe0cad9fae3d263 is 50, key is test_row_0/B:col10/1732466522523/Put/seqid=0 2024-11-24T16:42:04,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742193_1369 (size=12001) 2024-11-24T16:42:04,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/86640e5b1074454fafe0cad9fae3d263 2024-11-24T16:42:04,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d30eed588e11498d9f9caf2bc2fb418d is 50, key is test_row_0/C:col10/1732466522523/Put/seqid=0 2024-11-24T16:42:04,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742194_1370 (size=12001) 2024-11-24T16:42:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T16:42:04,364 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:04,365 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-24T16:42:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T16:42:04,367 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:04,367 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:04,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:04,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d30eed588e11498d9f9caf2bc2fb418d 2024-11-24T16:42:04,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/015275c0b9b64ca198d6d5233e1c0535 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/015275c0b9b64ca198d6d5233e1c0535 2024-11-24T16:42:04,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/015275c0b9b64ca198d6d5233e1c0535, entries=200, sequenceid=60, filesize=38.6 K 2024-11-24T16:42:04,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/86640e5b1074454fafe0cad9fae3d263 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/86640e5b1074454fafe0cad9fae3d263 2024-11-24T16:42:04,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/86640e5b1074454fafe0cad9fae3d263, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T16:42:04,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d30eed588e11498d9f9caf2bc2fb418d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d30eed588e11498d9f9caf2bc2fb418d 2024-11-24T16:42:04,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d30eed588e11498d9f9caf2bc2fb418d, entries=150, sequenceid=60, filesize=11.7 K 2024-11-24T16:42:04,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3abd8569b30f1874fc795b3a06786690 in 1268ms, sequenceid=60, compaction requested=true 2024-11-24T16:42:04,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:04,438 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:04,438 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:04,440 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:04,440 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:04,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T16:42:04,440 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,440 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/cca2c61bb50b4f77b85ad07f0af6d259, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9f42a5145c52427d860c7cfd12b2ca92, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/015275c0b9b64ca198d6d5233e1c0535] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=99.1 K 2024-11-24T16:42:04,440 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,440 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/cca2c61bb50b4f77b85ad07f0af6d259, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9f42a5145c52427d860c7cfd12b2ca92, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/015275c0b9b64ca198d6d5233e1c0535] 2024-11-24T16:42:04,441 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:04,441 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:04,441 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,441 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a3a3dda78e724940ab0d95f97971ebcd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/6f14784ea3c34414a79d0ff5318b5a4c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/86640e5b1074454fafe0cad9fae3d263] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=35.2 K 2024-11-24T16:42:04,442 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting cca2c61bb50b4f77b85ad07f0af6d259, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732466520210 2024-11-24T16:42:04,442 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a3a3dda78e724940ab0d95f97971ebcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732466520210 2024-11-24T16:42:04,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:04,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:04,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:04,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:04,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:04,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:04,444 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f14784ea3c34414a79d0ff5318b5a4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732466520308 2024-11-24T16:42:04,444 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f42a5145c52427d860c7cfd12b2ca92, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732466520308 2024-11-24T16:42:04,444 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 86640e5b1074454fafe0cad9fae3d263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466522522 2024-11-24T16:42:04,445 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 015275c0b9b64ca198d6d5233e1c0535, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466522520 2024-11-24T16:42:04,454 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:04,455 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/9d01815c18e14cc3b2af757f2172e431 is 50, key is test_row_0/B:col10/1732466522523/Put/seqid=0 2024-11-24T16:42:04,457 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:04,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240ed62ad83eb9463cb707bf861fda1479_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466523264/Put/seqid=0 2024-11-24T16:42:04,461 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411241a7ce269fbc84ff79edac3cefd9c6508_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:04,464 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411241a7ce269fbc84ff79edac3cefd9c6508_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:04,464 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241a7ce269fbc84ff79edac3cefd9c6508_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466584458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466584459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466584466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T16:42:04,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742196_1372 (size=14594) 2024-11-24T16:42:04,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742195_1371 (size=12104) 2024-11-24T16:42:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466584467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742197_1373 (size=4469) 2024-11-24T16:42:04,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466584553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,562 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 3abd8569b30f1874fc795b3a06786690 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:04,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466584574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466584574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466584574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466584593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:04,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:04,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T16:42:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466584784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466584784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466584793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466584814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:04,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,883 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:04,886 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240ed62ad83eb9463cb707bf861fda1479_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240ed62ad83eb9463cb707bf861fda1479_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:04,887 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/2fb0514132504dbbac38fac24a79aeff, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:04,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/2fb0514132504dbbac38fac24a79aeff is 175, key is test_row_0/A:col10/1732466523264/Put/seqid=0 2024-11-24T16:42:04,889 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/9d01815c18e14cc3b2af757f2172e431 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/9d01815c18e14cc3b2af757f2172e431 2024-11-24T16:42:04,891 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:04,892 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/a6c7215a24df47249d4d9c99e45e0e22 is 175, key is test_row_0/A:col10/1732466522523/Put/seqid=0 2024-11-24T16:42:04,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742198_1374 (size=39549) 2024-11-24T16:42:04,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742199_1375 (size=31058) 2024-11-24T16:42:04,905 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into 9d01815c18e14cc3b2af757f2172e431(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:04,905 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:04,905 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=13, startTime=1732466524438; duration=0sec 2024-11-24T16:42:04,906 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:04,906 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:04,906 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:04,906 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:04,906 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:04,907 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,907 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/4ec904296eab40fa8d015ea837dda863, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/44a8d38fdb384de1838e4beba59c151f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d30eed588e11498d9f9caf2bc2fb418d] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=35.2 K 2024-11-24T16:42:04,907 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ec904296eab40fa8d015ea837dda863, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732466520210 2024-11-24T16:42:04,907 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 44a8d38fdb384de1838e4beba59c151f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732466520308 2024-11-24T16:42:04,908 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d30eed588e11498d9f9caf2bc2fb418d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466522522 2024-11-24T16:42:04,915 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:04,916 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/93c07a2ef63c4648becc57d7e57f4ffd is 50, key is test_row_0/C:col10/1732466522523/Put/seqid=0 2024-11-24T16:42:04,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742200_1376 (size=12104) 2024-11-24T16:42:04,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:04,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T16:42:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466585104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466585104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466585113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466585134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/2fb0514132504dbbac38fac24a79aeff 2024-11-24T16:42:05,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/4ce6a8d962d24f91891aa94714f4eebd is 50, key is test_row_0/B:col10/1732466523264/Put/seqid=0 2024-11-24T16:42:05,307 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/a6c7215a24df47249d4d9c99e45e0e22 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/a6c7215a24df47249d4d9c99e45e0e22 2024-11-24T16:42:05,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742201_1377 (size=12001) 2024-11-24T16:42:05,312 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into a6c7215a24df47249d4d9c99e45e0e22(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:05,312 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:05,312 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=13, startTime=1732466524437; duration=0sec 2024-11-24T16:42:05,312 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:05,312 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:05,331 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/93c07a2ef63c4648becc57d7e57f4ffd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/93c07a2ef63c4648becc57d7e57f4ffd 2024-11-24T16:42:05,339 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into 93c07a2ef63c4648becc57d7e57f4ffd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:05,339 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:05,339 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=13, startTime=1732466524438; duration=0sec 2024-11-24T16:42:05,339 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:05,339 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:05,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:05,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:05,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T16:42:05,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466585623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466585634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466585634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466585653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:05,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/4ce6a8d962d24f91891aa94714f4eebd 2024-11-24T16:42:05,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/577c3a8429344692b297461eb9aefcc1 is 50, key is test_row_0/C:col10/1732466523264/Put/seqid=0 2024-11-24T16:42:05,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742202_1378 (size=12001) 2024-11-24T16:42:05,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:05,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:06,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:06,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:06,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/577c3a8429344692b297461eb9aefcc1 2024-11-24T16:42:06,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/2fb0514132504dbbac38fac24a79aeff as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/2fb0514132504dbbac38fac24a79aeff 2024-11-24T16:42:06,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/2fb0514132504dbbac38fac24a79aeff, entries=200, sequenceid=96, filesize=38.6 K 2024-11-24T16:42:06,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/4ce6a8d962d24f91891aa94714f4eebd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/4ce6a8d962d24f91891aa94714f4eebd 2024-11-24T16:42:06,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/4ce6a8d962d24f91891aa94714f4eebd, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T16:42:06,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/577c3a8429344692b297461eb9aefcc1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/577c3a8429344692b297461eb9aefcc1 2024-11-24T16:42:06,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/577c3a8429344692b297461eb9aefcc1, entries=150, sequenceid=96, filesize=11.7 K 2024-11-24T16:42:06,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 3abd8569b30f1874fc795b3a06786690 in 1708ms, sequenceid=96, compaction requested=false 2024-11-24T16:42:06,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:06,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T16:42:06,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:06,205 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T16:42:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:06,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243d320f5846b144199a4e8d66afc66ab4_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466524464/Put/seqid=0 2024-11-24T16:42:06,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742203_1379 (size=12154) 2024-11-24T16:42:06,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:06,224 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243d320f5846b144199a4e8d66afc66ab4_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243d320f5846b144199a4e8d66afc66ab4_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:06,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/7bc55378b0cb4c6abeeb1f15ddd0868e, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/7bc55378b0cb4c6abeeb1f15ddd0868e is 175, key is test_row_0/A:col10/1732466524464/Put/seqid=0 2024-11-24T16:42:06,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742204_1380 (size=30955) 2024-11-24T16:42:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T16:42:06,630 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/7bc55378b0cb4c6abeeb1f15ddd0868e 2024-11-24T16:42:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/2026a024e0774df89f6be28dae835bae is 50, key is test_row_0/B:col10/1732466524464/Put/seqid=0 2024-11-24T16:42:06,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742205_1381 (size=12001) 2024-11-24T16:42:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:06,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:06,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466526727, max timestamp=1732466526744Num uniques 859; 2024-11-24T16:42:06,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:42:06,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466526727, max timestamp=1732466526744Num uniques 859; 2024-11-24T16:42:06,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:42:06,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466526727, max timestamp=1732466526744Num uniques 859; 2024-11-24T16:42:06,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:42:06,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466586794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466586798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:06,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466586799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466586801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:06,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466586914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466586914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:06,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466586923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466586923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:06,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,078 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/2026a024e0774df89f6be28dae835bae 2024-11-24T16:42:07,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3dcd2eaea4074906a6c92ab166c59a8c is 50, key is test_row_0/C:col10/1732466524464/Put/seqid=0 2024-11-24T16:42:07,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742206_1382 (size=12001) 2024-11-24T16:42:07,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466587134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466587134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466587134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466587143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466587454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466587454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466587455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466587455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,490 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3dcd2eaea4074906a6c92ab166c59a8c 2024-11-24T16:42:07,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/7bc55378b0cb4c6abeeb1f15ddd0868e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/7bc55378b0cb4c6abeeb1f15ddd0868e 2024-11-24T16:42:07,498 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/7bc55378b0cb4c6abeeb1f15ddd0868e, entries=150, sequenceid=108, filesize=30.2 K 2024-11-24T16:42:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/2026a024e0774df89f6be28dae835bae as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2026a024e0774df89f6be28dae835bae 2024-11-24T16:42:07,502 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2026a024e0774df89f6be28dae835bae, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T16:42:07,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3dcd2eaea4074906a6c92ab166c59a8c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3dcd2eaea4074906a6c92ab166c59a8c 2024-11-24T16:42:07,506 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3dcd2eaea4074906a6c92ab166c59a8c, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T16:42:07,506 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for 3abd8569b30f1874fc795b3a06786690 in 1301ms, sequenceid=108, compaction requested=true 2024-11-24T16:42:07,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:07,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:07,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-24T16:42:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-24T16:42:07,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-24T16:42:07,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1410 sec 2024-11-24T16:42:07,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.1450 sec 2024-11-24T16:42:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:07,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-24T16:42:07,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:07,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:07,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:07,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:07,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:07,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:07,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b9d9219ee950438087d1ed19b8ccf173_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466526799/Put/seqid=0 2024-11-24T16:42:07,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742207_1383 (size=14794) 2024-11-24T16:42:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466587988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466587989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466587990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466587992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466588104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466588104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466588114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466588114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466588323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466588324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466588334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466588344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b9d9219ee950438087d1ed19b8ccf173_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9d9219ee950438087d1ed19b8ccf173_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:08,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/b557672f3e91496ea18ee250b860f2da, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:08,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/b557672f3e91496ea18ee250b860f2da is 175, key is test_row_0/A:col10/1732466526799/Put/seqid=0 2024-11-24T16:42:08,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742208_1384 (size=39749) 2024-11-24T16:42:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T16:42:08,544 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:08,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-24T16:42:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T16:42:08,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:08,547 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:08,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:08,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466588584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,594 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 3abd8569b30f1874fc795b3a06786690 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T16:42:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466588643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466588644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466588644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:08,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466588664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:08,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:08,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T16:42:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:08,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:08,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/b557672f3e91496ea18ee250b860f2da 2024-11-24T16:42:08,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/7621b8bc2241434ca8b8bffd12e65e09 is 50, key is test_row_0/B:col10/1732466526799/Put/seqid=0 2024-11-24T16:42:08,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742209_1385 (size=12151) 2024-11-24T16:42:08,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/7621b8bc2241434ca8b8bffd12e65e09 2024-11-24T16:42:08,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3e297de3aab64ae8b37460325655ae95 is 50, key is test_row_0/C:col10/1732466526799/Put/seqid=0 2024-11-24T16:42:08,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742210_1386 (size=12151) 2024-11-24T16:42:08,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3e297de3aab64ae8b37460325655ae95 2024-11-24T16:42:08,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/b557672f3e91496ea18ee250b860f2da as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/b557672f3e91496ea18ee250b860f2da 2024-11-24T16:42:08,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/b557672f3e91496ea18ee250b860f2da, entries=200, sequenceid=145, filesize=38.8 K 2024-11-24T16:42:08,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/7621b8bc2241434ca8b8bffd12e65e09 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7621b8bc2241434ca8b8bffd12e65e09 2024-11-24T16:42:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7621b8bc2241434ca8b8bffd12e65e09, entries=150, sequenceid=145, filesize=11.9 K 2024-11-24T16:42:08,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3e297de3aab64ae8b37460325655ae95 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3e297de3aab64ae8b37460325655ae95 2024-11-24T16:42:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3e297de3aab64ae8b37460325655ae95, entries=150, sequenceid=145, filesize=11.9 K 2024-11-24T16:42:08,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T16:42:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 3abd8569b30f1874fc795b3a06786690 in 875ms, sequenceid=145, compaction requested=true 2024-11-24T16:42:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:08,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:08,852 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:08,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:08,852 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,854 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141311 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:08,854 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:08,854 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:08,854 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:08,854 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,854 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,854 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/9d01815c18e14cc3b2af757f2172e431, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/4ce6a8d962d24f91891aa94714f4eebd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2026a024e0774df89f6be28dae835bae, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7621b8bc2241434ca8b8bffd12e65e09] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=47.1 K 2024-11-24T16:42:08,854 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/a6c7215a24df47249d4d9c99e45e0e22, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/2fb0514132504dbbac38fac24a79aeff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/7bc55378b0cb4c6abeeb1f15ddd0868e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/b557672f3e91496ea18ee250b860f2da] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=138.0 K 2024-11-24T16:42:08,854 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,854 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/a6c7215a24df47249d4d9c99e45e0e22, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/2fb0514132504dbbac38fac24a79aeff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/7bc55378b0cb4c6abeeb1f15ddd0868e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/b557672f3e91496ea18ee250b860f2da] 2024-11-24T16:42:08,855 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c7215a24df47249d4d9c99e45e0e22, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466522522 2024-11-24T16:42:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,855 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb0514132504dbbac38fac24a79aeff, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466523236 2024-11-24T16:42:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,855 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d01815c18e14cc3b2af757f2172e431, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466522522 2024-11-24T16:42:08,855 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bc55378b0cb4c6abeeb1f15ddd0868e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732466524445 2024-11-24T16:42:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,855 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce6a8d962d24f91891aa94714f4eebd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466523236 2024-11-24T16:42:08,855 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b557672f3e91496ea18ee250b860f2da, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732466526791 2024-11-24T16:42:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,856 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 2026a024e0774df89f6be28dae835bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732466524445 2024-11-24T16:42:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,856 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7621b8bc2241434ca8b8bffd12e65e09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732466526798 2024-11-24T16:42:08,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T16:42:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,865 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,883 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,884 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/55492f428bab4a78a72acef957bfd968 is 50, key is test_row_0/B:col10/1732466526799/Put/seqid=0 2024-11-24T16:42:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,886 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112455b1c02e31a34ba587a5cb3c1677ad38_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,888 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112455b1c02e31a34ba587a5cb3c1677ad38_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:08,888 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112455b1c02e31a34ba587a5cb3c1677ad38_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742211_1387 (size=12391) 2024-11-24T16:42:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,923 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/55492f428bab4a78a72acef957bfd968 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55492f428bab4a78a72acef957bfd968 2024-11-24T16:42:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742212_1388 (size=4469) 2024-11-24T16:42:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,933 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into 55492f428bab4a78a72acef957bfd968(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:08,933 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:08,933 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=12, startTime=1732466528852; duration=0sec 2024-11-24T16:42:08,933 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:08,933 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,933 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,935 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:08,935 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:08,935 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:08,935 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/93c07a2ef63c4648becc57d7e57f4ffd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/577c3a8429344692b297461eb9aefcc1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3dcd2eaea4074906a6c92ab166c59a8c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3e297de3aab64ae8b37460325655ae95] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=47.1 K 2024-11-24T16:42:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,936 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 93c07a2ef63c4648becc57d7e57f4ffd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732466522522 2024-11-24T16:42:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,936 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 577c3a8429344692b297461eb9aefcc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732466523236 2024-11-24T16:42:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,936 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dcd2eaea4074906a6c92ab166c59a8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732466524445 2024-11-24T16:42:08,937 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e297de3aab64ae8b37460325655ae95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732466526798 2024-11-24T16:42:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,947 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,948 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/eff80df9ecfd4ec4981cccae8166dbec is 50, key is test_row_0/C:col10/1732466526799/Put/seqid=0 2024-11-24T16:42:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742213_1389 (size=12391) 2024-11-24T16:42:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T16:42:09,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:09,006 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T16:42:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:09,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112431064545bb3c4bff99b8f7f43fc659a2_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466527988/Put/seqid=0 2024-11-24T16:42:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742214_1390 (size=9814) 2024-11-24T16:42:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T16:42:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:09,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466529245, max timestamp=1732466529281Num uniques 859; 2024-11-24T16:42:09,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:42:09,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466529245, max timestamp=1732466529281Num uniques 859; 2024-11-24T16:42:09,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:42:09,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732466529245, max timestamp=1732466529281Num uniques 859; 2024-11-24T16:42:09,288 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T16:42:09,327 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#332 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:09,327 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/4f9988cfbc9740beb25edabb307c7430 is 175, key is test_row_0/A:col10/1732466526799/Put/seqid=0 2024-11-24T16:42:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742215_1391 (size=31345) 2024-11-24T16:42:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466589323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466589324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466589338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,357 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/eff80df9ecfd4ec4981cccae8166dbec as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/eff80df9ecfd4ec4981cccae8166dbec 2024-11-24T16:42:09,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466589340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,362 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into eff80df9ecfd4ec4981cccae8166dbec(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:09,362 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:09,362 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=12, startTime=1732466528852; duration=0sec 2024-11-24T16:42:09,362 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:09,362 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:09,442 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112431064545bb3c4bff99b8f7f43fc659a2_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112431064545bb3c4bff99b8f7f43fc659a2_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/0f55aab357cf41a9bd4bc4fe1ca9bdda, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:09,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/0f55aab357cf41a9bd4bc4fe1ca9bdda is 175, key is test_row_0/A:col10/1732466527988/Put/seqid=0 2024-11-24T16:42:09,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742216_1392 (size=22461) 2024-11-24T16:42:09,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466589454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466589454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466589454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466589463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466589664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466589664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466589664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T16:42:09,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466589684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,736 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/4f9988cfbc9740beb25edabb307c7430 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4f9988cfbc9740beb25edabb307c7430 2024-11-24T16:42:09,741 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into 4f9988cfbc9740beb25edabb307c7430(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:09,741 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:09,741 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=12, startTime=1732466528852; duration=0sec 2024-11-24T16:42:09,741 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:09,741 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:09,854 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/0f55aab357cf41a9bd4bc4fe1ca9bdda 2024-11-24T16:42:09,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/10b7342b31ae4208be4d426aaaf5fcbf is 50, key is test_row_0/B:col10/1732466527988/Put/seqid=0 2024-11-24T16:42:09,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742217_1393 (size=9757) 2024-11-24T16:42:09,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466589974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466589974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466589974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:09,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:10,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466590004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:10,265 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/10b7342b31ae4208be4d426aaaf5fcbf 2024-11-24T16:42:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/af7195301fe74885a726f54426a49eee is 50, key is test_row_0/C:col10/1732466527988/Put/seqid=0 2024-11-24T16:42:10,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742218_1394 (size=9757) 2024-11-24T16:42:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466590493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466590493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:10,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466590493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466590524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:10,679 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/af7195301fe74885a726f54426a49eee 2024-11-24T16:42:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/0f55aab357cf41a9bd4bc4fe1ca9bdda as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/0f55aab357cf41a9bd4bc4fe1ca9bdda 2024-11-24T16:42:10,687 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/0f55aab357cf41a9bd4bc4fe1ca9bdda, entries=100, sequenceid=154, filesize=21.9 K 2024-11-24T16:42:10,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/10b7342b31ae4208be4d426aaaf5fcbf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/10b7342b31ae4208be4d426aaaf5fcbf 2024-11-24T16:42:10,691 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/10b7342b31ae4208be4d426aaaf5fcbf, entries=100, sequenceid=154, filesize=9.5 K 2024-11-24T16:42:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/af7195301fe74885a726f54426a49eee as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/af7195301fe74885a726f54426a49eee 2024-11-24T16:42:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T16:42:10,699 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/af7195301fe74885a726f54426a49eee, entries=100, sequenceid=154, filesize=9.5 K 2024-11-24T16:42:10,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 3abd8569b30f1874fc795b3a06786690 in 1695ms, sequenceid=154, compaction requested=false 2024-11-24T16:42:10,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:10,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:10,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-24T16:42:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-24T16:42:10,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-24T16:42:10,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1540 sec 2024-11-24T16:42:10,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1600 sec 2024-11-24T16:42:11,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:11,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-24T16:42:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:11,511 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:11,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240daa8eb8f97448d2b9da7c830ffa7944_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466531509/Put/seqid=0 2024-11-24T16:42:11,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742219_1395 (size=14794) 2024-11-24T16:42:11,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466591518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466591526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466591527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466591543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,555 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 3abd8569b30f1874fc795b3a06786690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466591633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466591634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466591633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466591853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466591853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466591853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:11,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:11,924 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:11,928 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240daa8eb8f97448d2b9da7c830ffa7944_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240daa8eb8f97448d2b9da7c830ffa7944_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:11,929 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/6ca4607ceaa447f6a76d3b69a1f7ca60, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:11,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/6ca4607ceaa447f6a76d3b69a1f7ca60 is 175, key is test_row_0/A:col10/1732466531509/Put/seqid=0 2024-11-24T16:42:11,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742220_1396 (size=39749) 2024-11-24T16:42:12,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466592164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:12,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466592164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466592164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:12,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/6ca4607ceaa447f6a76d3b69a1f7ca60 2024-11-24T16:42:12,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d95f648715ee42909ad0830aad40df0c is 50, key is test_row_0/B:col10/1732466531509/Put/seqid=0 2024-11-24T16:42:12,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742221_1397 (size=12151) 2024-11-24T16:42:12,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466592684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:12,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:12,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466592694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:12,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466592694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:12,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T16:42:12,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:12,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-24T16:42:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T16:42:12,716 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:12,717 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:12,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:12,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d95f648715ee42909ad0830aad40df0c 2024-11-24T16:42:12,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/1a48b0c4599249ba98c7b0b4713823da is 50, key is test_row_0/C:col10/1732466531509/Put/seqid=0 2024-11-24T16:42:12,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742222_1398 (size=12151) 2024-11-24T16:42:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T16:42:12,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T16:42:12,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:12,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:12,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:13,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T16:42:13,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:13,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:13,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:13,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T16:42:13,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/1a48b0c4599249ba98c7b0b4713823da 2024-11-24T16:42:13,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/6ca4607ceaa447f6a76d3b69a1f7ca60 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/6ca4607ceaa447f6a76d3b69a1f7ca60 2024-11-24T16:42:13,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/6ca4607ceaa447f6a76d3b69a1f7ca60, entries=200, sequenceid=194, filesize=38.8 K 2024-11-24T16:42:13,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d95f648715ee42909ad0830aad40df0c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d95f648715ee42909ad0830aad40df0c 2024-11-24T16:42:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d95f648715ee42909ad0830aad40df0c, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T16:42:13,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/1a48b0c4599249ba98c7b0b4713823da as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1a48b0c4599249ba98c7b0b4713823da 2024-11-24T16:42:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1a48b0c4599249ba98c7b0b4713823da, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T16:42:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~451.27 KB/462096, currentSize=26.84 KB/27480 for 3abd8569b30f1874fc795b3a06786690 in 1666ms, sequenceid=194, compaction requested=true 2024-11-24T16:42:13,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T16:42:13,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,176 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,176 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T16:42:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:13,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:13,176 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:13,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:13,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:13,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:13,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:13,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,177 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:13,177 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93555 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:13,178 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:13,178 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,178 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,178 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,178 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4f9988cfbc9740beb25edabb307c7430, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/0f55aab357cf41a9bd4bc4fe1ca9bdda, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/6ca4607ceaa447f6a76d3b69a1f7ca60] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=91.4 K 2024-11-24T16:42:13,178 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55492f428bab4a78a72acef957bfd968, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/10b7342b31ae4208be4d426aaaf5fcbf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d95f648715ee42909ad0830aad40df0c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=33.5 K 2024-11-24T16:42:13,178 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,178 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4f9988cfbc9740beb25edabb307c7430, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/0f55aab357cf41a9bd4bc4fe1ca9bdda, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/6ca4607ceaa447f6a76d3b69a1f7ca60] 2024-11-24T16:42:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,178 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 55492f428bab4a78a72acef957bfd968, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732466526798 2024-11-24T16:42:13,178 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f9988cfbc9740beb25edabb307c7430, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732466526798 2024-11-24T16:42:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,179 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f55aab357cf41a9bd4bc4fe1ca9bdda, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732466527988 2024-11-24T16:42:13,179 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b7342b31ae4208be4d426aaaf5fcbf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732466527988 2024-11-24T16:42:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,179 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d95f648715ee42909ad0830aad40df0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466529317 2024-11-24T16:42:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,180 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ca4607ceaa447f6a76d3b69a1f7ca60, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466529317 2024-11-24T16:42:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c7134305170948f6880ed8d65221489d_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_1/A:col10/1732466531524/Put/seqid=0 2024-11-24T16:42:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,187 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,188 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,188 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/55f5e672dd404d9cb329385de6ab4a16 is 50, key is test_row_0/B:col10/1732466531509/Put/seqid=0 2024-11-24T16:42:13,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,189 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124ee65c96de8254bb0beee4898d293aee2_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,190 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124ee65c96de8254bb0beee4898d293aee2_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,191 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ee65c96de8254bb0beee4898d293aee2_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742223_1399 (size=9814) 2024-11-24T16:42:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742224_1400 (size=12493) 2024-11-24T16:42:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742225_1401 (size=4469) 2024-11-24T16:42:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T16:42:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,598 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c7134305170948f6880ed8d65221489d_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c7134305170948f6880ed8d65221489d_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/07b53b32ee3743eab5475af2ff34842a, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/07b53b32ee3743eab5475af2ff34842a is 175, key is test_row_1/A:col10/1732466531524/Put/seqid=0 2024-11-24T16:42:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,605 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,605 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/abe6937b48dc4c9db5b4121b241e1a71 is 175, key is test_row_0/A:col10/1732466531509/Put/seqid=0 2024-11-24T16:42:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,609 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/55f5e672dd404d9cb329385de6ab4a16 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55f5e672dd404d9cb329385de6ab4a16 2024-11-24T16:42:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742226_1402 (size=22461) 2024-11-24T16:42:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,612 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/07b53b32ee3743eab5475af2ff34842a 2024-11-24T16:42:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,618 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into 55f5e672dd404d9cb329385de6ab4a16(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:13,618 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:13,618 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=13, startTime=1732466533176; duration=0sec 2024-11-24T16:42:13,618 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:13,618 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:13,618 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,620 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:13,620 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:13,620 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,620 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/eff80df9ecfd4ec4981cccae8166dbec, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/af7195301fe74885a726f54426a49eee, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1a48b0c4599249ba98c7b0b4713823da] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=33.5 K 2024-11-24T16:42:13,620 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting eff80df9ecfd4ec4981cccae8166dbec, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732466526798 2024-11-24T16:42:13,621 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting af7195301fe74885a726f54426a49eee, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732466527988 2024-11-24T16:42:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,621 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a48b0c4599249ba98c7b0b4713823da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466529317 2024-11-24T16:42:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/94e0bf18085543be8956aadcf5b3267a is 50, key is test_row_1/B:col10/1732466531524/Put/seqid=0 2024-11-24T16:42:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742227_1403 (size=31447) 2024-11-24T16:42:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,647 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:13,647 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/6a4c2aeba12f4c2583bef1be7ceba549 is 50, key is test_row_0/C:col10/1732466531509/Put/seqid=0 2024-11-24T16:42:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742229_1405 (size=12493) 2024-11-24T16:42:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,664 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/6a4c2aeba12f4c2583bef1be7ceba549 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/6a4c2aeba12f4c2583bef1be7ceba549 2024-11-24T16:42:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,669 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into 6a4c2aeba12f4c2583bef1be7ceba549(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:13,669 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:13,669 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=13, startTime=1732466533177; duration=0sec 2024-11-24T16:42:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,669 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:13,669 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742228_1404 (size=9757) 2024-11-24T16:42:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,680 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/94e0bf18085543be8956aadcf5b3267a 2024-11-24T16:42:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d0cd2767f81a4e819d4558f7bac94732 is 50, key is test_row_1/C:col10/1732466531524/Put/seqid=0 2024-11-24T16:42:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742230_1406 (size=9757) 2024-11-24T16:42:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,713 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d0cd2767f81a4e819d4558f7bac94732 2024-11-24T16:42:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/07b53b32ee3743eab5475af2ff34842a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/07b53b32ee3743eab5475af2ff34842a 2024-11-24T16:42:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,722 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/07b53b32ee3743eab5475af2ff34842a, entries=100, sequenceid=201, filesize=21.9 K 2024-11-24T16:42:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/94e0bf18085543be8956aadcf5b3267a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/94e0bf18085543be8956aadcf5b3267a 2024-11-24T16:42:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,728 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/94e0bf18085543be8956aadcf5b3267a, entries=100, sequenceid=201, filesize=9.5 K 2024-11-24T16:42:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d0cd2767f81a4e819d4558f7bac94732 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d0cd2767f81a4e819d4558f7bac94732 2024-11-24T16:42:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,736 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d0cd2767f81a4e819d4558f7bac94732, entries=100, sequenceid=201, filesize=9.5 K 2024-11-24T16:42:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,737 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=40.25 KB/41220 for 3abd8569b30f1874fc795b3a06786690 in 561ms, sequenceid=201, compaction requested=false 2024-11-24T16:42:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-24T16:42:13,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:13,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-24T16:42:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-24T16:42:13,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0220 sec 2024-11-24T16:42:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0290 sec 2024-11-24T16:42:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247e7bbeeffbcc43df94e62a6b814253c7_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466533734/Put/seqid=0 2024-11-24T16:42:13,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742231_1407 (size=22268) 2024-11-24T16:42:13,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:13,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247e7bbeeffbcc43df94e62a6b814253c7_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247e7bbeeffbcc43df94e62a6b814253c7_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:13,797 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/144da2df2c494edab6936975a05302ac, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:13,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/144da2df2c494edab6936975a05302ac is 175, key is test_row_0/A:col10/1732466533734/Put/seqid=0 2024-11-24T16:42:13,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742232_1408 (size=65673) 2024-11-24T16:42:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T16:42:13,854 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:13,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:13,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466533787, max timestamp=1732466533850Num uniques 859; 2024-11-24T16:42:13,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:13,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466533787, max timestamp=1732466533850Num uniques 859; 2024-11-24T16:42:13,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466533787, max timestamp=1732466533850Num uniques 859; 2024-11-24T16:42:13,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:13,855 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-24T16:42:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T16:42:13,857 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:13,857 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:13,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:13,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466593902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466593922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:13,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466593922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466593923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:13,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T16:42:14,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:14,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:14,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,044 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/abe6937b48dc4c9db5b4121b241e1a71 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/abe6937b48dc4c9db5b4121b241e1a71 2024-11-24T16:42:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466594034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:14,050 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into abe6937b48dc4c9db5b4121b241e1a71(size=30.7 K), total size for store is 52.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,050 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,050 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=13, startTime=1732466533176; duration=0sec 2024-11-24T16:42:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:14,050 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:14,050 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:14,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466594053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:14,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466594053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:14,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466594054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:14,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:14,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T16:42:14,208 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/144da2df2c494edab6936975a05302ac 2024-11-24T16:42:14,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d665d2f27dcd4171bd6cf2d8800b2899 is 50, key is test_row_0/B:col10/1732466533734/Put/seqid=0 2024-11-24T16:42:14,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742233_1409 (size=12151) 2024-11-24T16:42:14,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d665d2f27dcd4171bd6cf2d8800b2899 2024-11-24T16:42:14,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/7c7f3ba3fee7462c8a63a377f1387afb is 50, key is test_row_0/C:col10/1732466533734/Put/seqid=0 2024-11-24T16:42:14,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742234_1410 (size=12151) 2024-11-24T16:42:14,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/7c7f3ba3fee7462c8a63a377f1387afb 2024-11-24T16:42:14,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466594255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:14,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:14,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/144da2df2c494edab6936975a05302ac as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/144da2df2c494edab6936975a05302ac 2024-11-24T16:42:14,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/144da2df2c494edab6936975a05302ac, entries=350, sequenceid=213, filesize=64.1 K 2024-11-24T16:42:14,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d665d2f27dcd4171bd6cf2d8800b2899 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d665d2f27dcd4171bd6cf2d8800b2899 2024-11-24T16:42:14,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d665d2f27dcd4171bd6cf2d8800b2899, entries=150, sequenceid=213, filesize=11.9 K 2024-11-24T16:42:14,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/7c7f3ba3fee7462c8a63a377f1387afb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/7c7f3ba3fee7462c8a63a377f1387afb 2024-11-24T16:42:14,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/7c7f3ba3fee7462c8a63a377f1387afb, entries=150, sequenceid=213, filesize=11.9 K 2024-11-24T16:42:14,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3abd8569b30f1874fc795b3a06786690 in 560ms, sequenceid=213, compaction requested=true 2024-11-24T16:42:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:14,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:14,297 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:14,297 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:14,298 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119581 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:14,298 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:14,298 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:14,298 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:14,298 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,298 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,298 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/abe6937b48dc4c9db5b4121b241e1a71, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/07b53b32ee3743eab5475af2ff34842a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/144da2df2c494edab6936975a05302ac] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=116.8 K 2024-11-24T16:42:14,298 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55f5e672dd404d9cb329385de6ab4a16, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/94e0bf18085543be8956aadcf5b3267a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d665d2f27dcd4171bd6cf2d8800b2899] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=33.6 K 2024-11-24T16:42:14,298 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,298 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/abe6937b48dc4c9db5b4121b241e1a71, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/07b53b32ee3743eab5475af2ff34842a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/144da2df2c494edab6936975a05302ac] 2024-11-24T16:42:14,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T16:42:14,299 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f5e672dd404d9cb329385de6ab4a16, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466529317 2024-11-24T16:42:14,299 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting abe6937b48dc4c9db5b4121b241e1a71, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466529317 2024-11-24T16:42:14,299 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e0bf18085543be8956aadcf5b3267a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732466531524 2024-11-24T16:42:14,299 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07b53b32ee3743eab5475af2ff34842a, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732466531524 2024-11-24T16:42:14,300 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d665d2f27dcd4171bd6cf2d8800b2899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732466533733 2024-11-24T16:42:14,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466533895, max timestamp=1732466534298Num uniques 1068; 2024-11-24T16:42:14,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-24T16:42:14,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466533895, max timestamp=1732466534298Num uniques 1068; 2024-11-24T16:42:14,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-24T16:42:14,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466533895, max timestamp=1732466534298Num uniques 1068; 2024-11-24T16:42:14,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-24T16:42:14,304 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 144da2df2c494edab6936975a05302ac, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732466533619 2024-11-24T16:42:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:14,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:14,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,320 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:14,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,329 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124df6a8607356a49caa7f07280922a8213_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:14,331 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:14,331 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/b3916c80f1554fa8bff3e4be5c20384f is 50, key is test_row_0/B:col10/1732466533734/Put/seqid=0 2024-11-24T16:42:14,332 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124df6a8607356a49caa7f07280922a8213_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:14,332 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124df6a8607356a49caa7f07280922a8213_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:14,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c734325071b541988638c03e8b68f144_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466533921/Put/seqid=0 2024-11-24T16:42:14,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742235_1411 (size=4469) 2024-11-24T16:42:14,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742236_1412 (size=12595) 2024-11-24T16:42:14,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742237_1413 (size=14794) 2024-11-24T16:42:14,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466534346, max timestamp=1732466534383Num uniques 859; 2024-11-24T16:42:14,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:14,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466534346, max timestamp=1732466534383Num uniques 859; 2024-11-24T16:42:14,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466534346, max timestamp=1732466534383Num uniques 859; 2024-11-24T16:42:14,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:14,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T16:42:14,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466594480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:14,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466534457, max timestamp=1732466534499Num uniques 1118; 2024-11-24T16:42:14,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-24T16:42:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466534457, max timestamp=1732466534499Num uniques 1118; 2024-11-24T16:42:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732466534457, max timestamp=1732466534499Num uniques 1118; 2024-11-24T16:42:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-24T16:42:14,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-24T16:42:14,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:14,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:14,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732466534598, max timestamp=1732466534630Num uniques 1077; 2024-11-24T16:42:14,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-24T16:42:14,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466594621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:14,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732466534598, max timestamp=1732466534630Num uniques 1077; 2024-11-24T16:42:14,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-24T16:42:14,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732466534598, max timestamp=1732466534630Num uniques 1077; 2024-11-24T16:42:14,638 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-11-24T16:42:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:14,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:14,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732466534655, max timestamp=1732466534735Num uniques 1136; 2024-11-24T16:42:14,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-24T16:42:14,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732466534655, max timestamp=1732466534735Num uniques 1136; 2024-11-24T16:42:14,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-24T16:42:14,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732466534655, max timestamp=1732466534735Num uniques 1136; 2024-11-24T16:42:14,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=13096, suffix heap size=146112,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=286 2024-11-24T16:42:14,754 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#350 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:14,755 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/12dafb5b0de5499c8efb26440fd93980 is 175, key is test_row_0/A:col10/1732466533734/Put/seqid=0 2024-11-24T16:42:14,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:14,765 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c734325071b541988638c03e8b68f144_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c734325071b541988638c03e8b68f144_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:14,766 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9d2f53a25df44cc9b5eef33ee10e47ea, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:14,766 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/b3916c80f1554fa8bff3e4be5c20384f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/b3916c80f1554fa8bff3e4be5c20384f 2024-11-24T16:42:14,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9d2f53a25df44cc9b5eef33ee10e47ea is 175, key is test_row_0/A:col10/1732466533921/Put/seqid=0 2024-11-24T16:42:14,772 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into b3916c80f1554fa8bff3e4be5c20384f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:14,772 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:14,772 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=13, startTime=1732466534297; duration=0sec 2024-11-24T16:42:14,772 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:14,772 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:14,772 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:14,774 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:14,774 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:14,774 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,774 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/6a4c2aeba12f4c2583bef1be7ceba549, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d0cd2767f81a4e819d4558f7bac94732, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/7c7f3ba3fee7462c8a63a377f1387afb] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=33.6 K 2024-11-24T16:42:14,774 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a4c2aeba12f4c2583bef1be7ceba549, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732466529317 2024-11-24T16:42:14,775 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cd2767f81a4e819d4558f7bac94732, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1732466531524 2024-11-24T16:42:14,775 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7f3ba3fee7462c8a63a377f1387afb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732466533733 2024-11-24T16:42:14,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,799 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:14,800 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/445c63e30657406b955413a048a32821 is 50, key is test_row_0/C:col10/1732466533734/Put/seqid=0 2024-11-24T16:42:14,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742238_1414 (size=31549) 2024-11-24T16:42:14,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742239_1415 (size=39749) 2024-11-24T16:42:14,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9d2f53a25df44cc9b5eef33ee10e47ea 2024-11-24T16:42:14,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742240_1416 (size=12595) 2024-11-24T16:42:14,821 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/445c63e30657406b955413a048a32821 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/445c63e30657406b955413a048a32821 2024-11-24T16:42:14,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/bbffc9655fe147d08f1da3104b0421b1 is 50, key is test_row_0/B:col10/1732466533921/Put/seqid=0 2024-11-24T16:42:14,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742241_1417 (size=12151) 2024-11-24T16:42:14,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/bbffc9655fe147d08f1da3104b0421b1 2024-11-24T16:42:14,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/c49379567fd44c55a0ce582533d02d3f is 50, key is test_row_0/C:col10/1732466533921/Put/seqid=0 2024-11-24T16:42:14,844 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into 445c63e30657406b955413a048a32821(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:14,844 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:14,844 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=13, startTime=1732466534297; duration=0sec 2024-11-24T16:42:14,845 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:14,845 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732466534789, max timestamp=1732466534844Num uniques 1145; 2024-11-24T16:42:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-24T16:42:14,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732466534789, max timestamp=1732466534844Num uniques 1145; 2024-11-24T16:42:14,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732466534789, max timestamp=1732466534844Num uniques 1145; 2024-11-24T16:42:14,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-24T16:42:14,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-24T16:42:14,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742242_1418 (size=12151) 2024-11-24T16:42:14,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/c49379567fd44c55a0ce582533d02d3f 2024-11-24T16:42:14,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/9d2f53a25df44cc9b5eef33ee10e47ea as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9d2f53a25df44cc9b5eef33ee10e47ea 2024-11-24T16:42:14,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9d2f53a25df44cc9b5eef33ee10e47ea, entries=200, sequenceid=252, filesize=38.8 K 2024-11-24T16:42:14,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/bbffc9655fe147d08f1da3104b0421b1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/bbffc9655fe147d08f1da3104b0421b1 2024-11-24T16:42:14,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/bbffc9655fe147d08f1da3104b0421b1, entries=150, sequenceid=252, filesize=11.9 K 2024-11-24T16:42:14,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/c49379567fd44c55a0ce582533d02d3f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/c49379567fd44c55a0ce582533d02d3f 2024-11-24T16:42:14,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/c49379567fd44c55a0ce582533d02d3f, entries=150, sequenceid=252, filesize=11.9 K 2024-11-24T16:42:14,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=60.38 KB/61830 for 3abd8569b30f1874fc795b3a06786690 in 597ms, sequenceid=252, compaction requested=false 2024-11-24T16:42:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:14,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=67.09 KB heapSize=177.70 KB 2024-11-24T16:42:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:14,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b92c4017d4f04e60bd2d892b263f61ef_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466534893/Put/seqid=0 2024-11-24T16:42:14,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742243_1419 (size=20074) 2024-11-24T16:42:14,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:14,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:14,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:14,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T16:42:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466595066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466595067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466595068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466595076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,206 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/12dafb5b0de5499c8efb26440fd93980 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/12dafb5b0de5499c8efb26440fd93980 2024-11-24T16:42:15,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466595193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466595194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466595194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466595194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,222 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into 12dafb5b0de5499c8efb26440fd93980(size=30.8 K), total size for store is 69.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:15,222 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:15,222 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=13, startTime=1732466534297; duration=0sec 2024-11-24T16:42:15,222 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:15,222 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:15,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:15,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:15,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b92c4017d4f04e60bd2d892b263f61ef_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b92c4017d4f04e60bd2d892b263f61ef_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:15,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/60fcbf0ccce4428dacaf99ca6bb39f43, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:15,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/60fcbf0ccce4428dacaf99ca6bb39f43 is 175, key is test_row_0/A:col10/1732466534893/Put/seqid=0 2024-11-24T16:42:15,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742244_1420 (size=57333) 2024-11-24T16:42:15,365 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/60fcbf0ccce4428dacaf99ca6bb39f43 2024-11-24T16:42:15,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/30731cfaafd24145874a8cd9e42483de is 50, key is test_row_0/B:col10/1732466534893/Put/seqid=0 2024-11-24T16:42:15,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:15,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466595424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466595424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466595425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466595425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742245_1421 (size=12301) 2024-11-24T16:42:15,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:15,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:15,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:15,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:15,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466595745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466595745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466595745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466595745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:15,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:15,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:15,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/30731cfaafd24145874a8cd9e42483de 2024-11-24T16:42:15,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:15,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:15,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:15,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d41df726bb53492dbd3bc102c3a7fa80 is 50, key is test_row_0/C:col10/1732466534893/Put/seqid=0 2024-11-24T16:42:15,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742246_1422 (size=12301) 2024-11-24T16:42:15,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d41df726bb53492dbd3bc102c3a7fa80 2024-11-24T16:42:15,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/60fcbf0ccce4428dacaf99ca6bb39f43 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/60fcbf0ccce4428dacaf99ca6bb39f43 2024-11-24T16:42:15,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/60fcbf0ccce4428dacaf99ca6bb39f43, entries=300, sequenceid=348, filesize=56.0 K 2024-11-24T16:42:15,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/30731cfaafd24145874a8cd9e42483de as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/30731cfaafd24145874a8cd9e42483de 2024-11-24T16:42:15,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/30731cfaafd24145874a8cd9e42483de, entries=150, sequenceid=348, filesize=12.0 K 2024-11-24T16:42:15,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/d41df726bb53492dbd3bc102c3a7fa80 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d41df726bb53492dbd3bc102c3a7fa80 2024-11-24T16:42:15,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d41df726bb53492dbd3bc102c3a7fa80, entries=150, sequenceid=348, filesize=12.0 K 2024-11-24T16:42:15,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~177.66 KB/181920, currentSize=114.05 KB/116790 for 3abd8569b30f1874fc795b3a06786690 in 1064ms, sequenceid=348, compaction requested=true 2024-11-24T16:42:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:15,960 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T16:42:15,960 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:15,961 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128631 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:15,961 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:15,961 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:15,961 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:15,961 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,962 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/12dafb5b0de5499c8efb26440fd93980, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9d2f53a25df44cc9b5eef33ee10e47ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/60fcbf0ccce4428dacaf99ca6bb39f43] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=125.6 K 2024-11-24T16:42:15,962 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,962 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/12dafb5b0de5499c8efb26440fd93980, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9d2f53a25df44cc9b5eef33ee10e47ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/60fcbf0ccce4428dacaf99ca6bb39f43] 2024-11-24T16:42:15,964 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:15,965 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/b3916c80f1554fa8bff3e4be5c20384f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/bbffc9655fe147d08f1da3104b0421b1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/30731cfaafd24145874a8cd9e42483de] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=36.2 K 2024-11-24T16:42:15,965 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 12dafb5b0de5499c8efb26440fd93980, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732466533733 2024-11-24T16:42:15,965 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3916c80f1554fa8bff3e4be5c20384f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732466533733 2024-11-24T16:42:15,966 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d2f53a25df44cc9b5eef33ee10e47ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732466533918 2024-11-24T16:42:15,966 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbffc9655fe147d08f1da3104b0421b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732466533918 2024-11-24T16:42:15,966 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 60fcbf0ccce4428dacaf99ca6bb39f43, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732466534799 2024-11-24T16:42:15,966 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30731cfaafd24145874a8cd9e42483de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732466534886 2024-11-24T16:42:15,983 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:15,984 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/a45a417a49f44c29a4c738e3ffa6c3cf is 50, key is test_row_0/B:col10/1732466534893/Put/seqid=0 2024-11-24T16:42:15,993 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T16:42:16,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T16:42:16,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:16,015 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:42:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:16,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:16,033 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124a1309e4a010e43069f18c374cd66fae5_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:16,036 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124a1309e4a010e43069f18c374cd66fae5_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:16,036 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a1309e4a010e43069f18c374cd66fae5_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:16,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742247_1423 (size=12847) 2024-11-24T16:42:16,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112428af3b69e7ca4eefaf62cbfbaaab9544_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466535063/Put/seqid=0 2024-11-24T16:42:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742248_1424 (size=4469) 2024-11-24T16:42:16,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742249_1425 (size=12454) 2024-11-24T16:42:16,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,174 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112428af3b69e7ca4eefaf62cbfbaaab9544_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112428af3b69e7ca4eefaf62cbfbaaab9544_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:16,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/111f1f67c93d434bba50a156d16a2a9b, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:16,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/111f1f67c93d434bba50a156d16a2a9b is 175, key is test_row_0/A:col10/1732466535063/Put/seqid=0 2024-11-24T16:42:16,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742250_1426 (size=31255) 2024-11-24T16:42:16,199 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/111f1f67c93d434bba50a156d16a2a9b 2024-11-24T16:42:16,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/0ca394c92a594511929af3b68350e9da is 50, key is test_row_0/B:col10/1732466535063/Put/seqid=0 2024-11-24T16:42:16,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742251_1427 (size=12301) 2024-11-24T16:42:16,234 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/0ca394c92a594511929af3b68350e9da 2024-11-24T16:42:16,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ba39f2cacc9d4940be7f97eecdbe38ff is 50, key is test_row_0/C:col10/1732466535063/Put/seqid=0 2024-11-24T16:42:16,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742252_1428 (size=12301) 2024-11-24T16:42:16,260 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ba39f2cacc9d4940be7f97eecdbe38ff 2024-11-24T16:42:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/111f1f67c93d434bba50a156d16a2a9b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/111f1f67c93d434bba50a156d16a2a9b 2024-11-24T16:42:16,277 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/111f1f67c93d434bba50a156d16a2a9b, entries=150, sequenceid=369, filesize=30.5 K 2024-11-24T16:42:16,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/0ca394c92a594511929af3b68350e9da as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/0ca394c92a594511929af3b68350e9da 2024-11-24T16:42:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,283 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/0ca394c92a594511929af3b68350e9da, entries=150, sequenceid=369, filesize=12.0 K 2024-11-24T16:42:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ba39f2cacc9d4940be7f97eecdbe38ff as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba39f2cacc9d4940be7f97eecdbe38ff 2024-11-24T16:42:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba39f2cacc9d4940be7f97eecdbe38ff, entries=150, sequenceid=369, filesize=12.0 K 2024-11-24T16:42:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,289 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 3abd8569b30f1874fc795b3a06786690 in 274ms, sequenceid=369, compaction requested=true 2024-11-24T16:42:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:16,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:16,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-24T16:42:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-24T16:42:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-24T16:42:16,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4330 sec 2024-11-24T16:42:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.4400 sec 2024-11-24T16:42:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:16,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:16,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:16,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:16,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:16,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112447b4aca1e2ab416784846a3e2e2d3e97_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466536343/Put/seqid=0 2024-11-24T16:42:16,483 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/a45a417a49f44c29a4c738e3ffa6c3cf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a45a417a49f44c29a4c738e3ffa6c3cf 2024-11-24T16:42:16,488 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into a45a417a49f44c29a4c738e3ffa6c3cf(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:16,488 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:16,488 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=13, startTime=1732466535960; duration=0sec 2024-11-24T16:42:16,488 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:16,488 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:16,488 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:16,489 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49348 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:16,489 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:16,490 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:16,490 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/445c63e30657406b955413a048a32821, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/c49379567fd44c55a0ce582533d02d3f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d41df726bb53492dbd3bc102c3a7fa80, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba39f2cacc9d4940be7f97eecdbe38ff] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=48.2 K 2024-11-24T16:42:16,490 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 445c63e30657406b955413a048a32821, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732466533733 2024-11-24T16:42:16,490 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49379567fd44c55a0ce582533d02d3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732466533918 2024-11-24T16:42:16,491 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41df726bb53492dbd3bc102c3a7fa80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732466534886 2024-11-24T16:42:16,491 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba39f2cacc9d4940be7f97eecdbe38ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732466535063 2024-11-24T16:42:16,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742253_1429 (size=22618) 2024-11-24T16:42:16,523 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:16,524 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ff1f6e5134034b6fa04cde0b8bf82fab is 50, key is test_row_0/C:col10/1732466535063/Put/seqid=0 2024-11-24T16:42:16,551 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#360 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:16,551 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/445ba6af815c4a469537940e00e473d5 is 175, key is test_row_0/A:col10/1732466534893/Put/seqid=0 2024-11-24T16:42:16,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466536502, max timestamp=1732466536554Num uniques 859; 2024-11-24T16:42:16,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466536502, max timestamp=1732466536554Num uniques 859; 2024-11-24T16:42:16,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:16,558 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:16,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466536502, max timestamp=1732466536554Num uniques 859; 2024-11-24T16:42:16,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:16,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742254_1430 (size=12881) 2024-11-24T16:42:16,565 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ff1f6e5134034b6fa04cde0b8bf82fab as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ff1f6e5134034b6fa04cde0b8bf82fab 2024-11-24T16:42:16,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742255_1431 (size=31801) 2024-11-24T16:42:16,579 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into ff1f6e5134034b6fa04cde0b8bf82fab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:16,579 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:16,579 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=12, startTime=1732466535960; duration=0sec 2024-11-24T16:42:16,579 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:16,579 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:16,582 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/445ba6af815c4a469537940e00e473d5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/445ba6af815c4a469537940e00e473d5 2024-11-24T16:42:16,587 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into 445ba6af815c4a469537940e00e473d5(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:16,587 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:16,587 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=13, startTime=1732466535960; duration=0sec 2024-11-24T16:42:16,587 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:16,587 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:16,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466596627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466596629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466596634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466596643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466596755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466596755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466596765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466596765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:16,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:16,919 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112447b4aca1e2ab416784846a3e2e2d3e97_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112447b4aca1e2ab416784846a3e2e2d3e97_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:16,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8d4c100e662f45aea147a2b9ff4ef50e, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:16,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8d4c100e662f45aea147a2b9ff4ef50e is 175, key is test_row_0/A:col10/1732466536343/Put/seqid=0 2024-11-24T16:42:16,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742256_1432 (size=66023) 2024-11-24T16:42:16,924 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8d4c100e662f45aea147a2b9ff4ef50e 2024-11-24T16:42:16,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/5af8eb1b18544ab48100e28c28c31717 is 50, key is test_row_0/B:col10/1732466536343/Put/seqid=0 2024-11-24T16:42:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742257_1433 (size=12301) 2024-11-24T16:42:16,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/5af8eb1b18544ab48100e28c28c31717 2024-11-24T16:42:16,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/8da1d713bc7245c6a5eee9d0d9827e89 is 50, key is test_row_0/C:col10/1732466536343/Put/seqid=0 2024-11-24T16:42:16,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742258_1434 (size=12301) 2024-11-24T16:42:16,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/8da1d713bc7245c6a5eee9d0d9827e89 2024-11-24T16:42:16,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8d4c100e662f45aea147a2b9ff4ef50e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8d4c100e662f45aea147a2b9ff4ef50e 2024-11-24T16:42:16,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8d4c100e662f45aea147a2b9ff4ef50e, entries=350, sequenceid=379, filesize=64.5 K 2024-11-24T16:42:16,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/5af8eb1b18544ab48100e28c28c31717 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5af8eb1b18544ab48100e28c28c31717 2024-11-24T16:42:16,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5af8eb1b18544ab48100e28c28c31717, entries=150, sequenceid=379, filesize=12.0 K 2024-11-24T16:42:16,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/8da1d713bc7245c6a5eee9d0d9827e89 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/8da1d713bc7245c6a5eee9d0d9827e89 2024-11-24T16:42:16,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/8da1d713bc7245c6a5eee9d0d9827e89, entries=150, sequenceid=379, filesize=12.0 K 2024-11-24T16:42:16,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3abd8569b30f1874fc795b3a06786690 in 529ms, sequenceid=379, compaction requested=true 2024-11-24T16:42:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:16,976 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:16,976 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:16,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:16,977 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:16,977 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:16,977 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129079 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:16,977 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:16,977 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:16,977 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a45a417a49f44c29a4c738e3ffa6c3cf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/0ca394c92a594511929af3b68350e9da, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5af8eb1b18544ab48100e28c28c31717] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=36.6 K 2024-11-24T16:42:16,977 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:16,977 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/445ba6af815c4a469537940e00e473d5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/111f1f67c93d434bba50a156d16a2a9b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8d4c100e662f45aea147a2b9ff4ef50e] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=126.1 K 2024-11-24T16:42:16,977 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:16,977 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/445ba6af815c4a469537940e00e473d5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/111f1f67c93d434bba50a156d16a2a9b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8d4c100e662f45aea147a2b9ff4ef50e] 2024-11-24T16:42:16,979 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a45a417a49f44c29a4c738e3ffa6c3cf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732466534886 2024-11-24T16:42:16,980 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca394c92a594511929af3b68350e9da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732466535063 2024-11-24T16:42:16,980 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af8eb1b18544ab48100e28c28c31717, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732466536343 2024-11-24T16:42:16,980 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 445ba6af815c4a469537940e00e473d5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1732466534886 2024-11-24T16:42:16,988 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 111f1f67c93d434bba50a156d16a2a9b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732466535063 2024-11-24T16:42:16,988 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d4c100e662f45aea147a2b9ff4ef50e, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732466536341 2024-11-24T16:42:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:16,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T16:42:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:17,007 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:17,007 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:17,007 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/c15947b4b2654a45812b8f2447b8d4f4 is 50, key is test_row_0/B:col10/1732466536343/Put/seqid=0 2024-11-24T16:42:17,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466597004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466597009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466597011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466597015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124146895b6e99042f79e96d526b41cca41_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466536640/Put/seqid=0 2024-11-24T16:42:17,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742259_1435 (size=12949) 2024-11-24T16:42:17,056 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112485da7e5ba4b14726b65c7927ad70bff9_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:17,058 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112485da7e5ba4b14726b65c7927ad70bff9_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:17,058 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112485da7e5ba4b14726b65c7927ad70bff9_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:17,063 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/c15947b4b2654a45812b8f2447b8d4f4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/c15947b4b2654a45812b8f2447b8d4f4 2024-11-24T16:42:17,068 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into c15947b4b2654a45812b8f2447b8d4f4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:17,068 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:17,068 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=13, startTime=1732466536976; duration=0sec 2024-11-24T16:42:17,068 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:17,068 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:17,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T16:42:17,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T16:42:17,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T16:42:17,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. because compaction request was cancelled 2024-11-24T16:42:17,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742260_1436 (size=12454) 2024-11-24T16:42:17,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,092 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124146895b6e99042f79e96d526b41cca41_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124146895b6e99042f79e96d526b41cca41_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:17,093 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/4c7c7ff97db04507b6f04d6218e02f13, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:17,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/4c7c7ff97db04507b6f04d6218e02f13 is 175, key is test_row_0/A:col10/1732466536640/Put/seqid=0 2024-11-24T16:42:17,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742261_1437 (size=4469) 2024-11-24T16:42:17,103 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#369 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:17,103 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/078ff12fe8484270b8214c0e0cb46a9c is 175, key is test_row_0/A:col10/1732466536343/Put/seqid=0 2024-11-24T16:42:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742262_1438 (size=31255) 2024-11-24T16:42:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742263_1439 (size=31903) 2024-11-24T16:42:17,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/4c7c7ff97db04507b6f04d6218e02f13 2024-11-24T16:42:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466597123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3e7c74cd51084657a4cbadf00ce766b8 is 50, key is test_row_0/B:col10/1732466536640/Put/seqid=0 2024-11-24T16:42:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742264_1440 (size=12301) 2024-11-24T16:42:17,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3e7c74cd51084657a4cbadf00ce766b8 2024-11-24T16:42:17,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3323dd4f201f4f068275bedd183be29b is 50, key is test_row_0/C:col10/1732466536640/Put/seqid=0 2024-11-24T16:42:17,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466597144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466597144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466597144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742265_1441 (size=12301) 2024-11-24T16:42:17,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3323dd4f201f4f068275bedd183be29b 2024-11-24T16:42:17,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/4c7c7ff97db04507b6f04d6218e02f13 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4c7c7ff97db04507b6f04d6218e02f13 2024-11-24T16:42:17,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4c7c7ff97db04507b6f04d6218e02f13, entries=150, sequenceid=420, filesize=30.5 K 2024-11-24T16:42:17,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3e7c74cd51084657a4cbadf00ce766b8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3e7c74cd51084657a4cbadf00ce766b8 2024-11-24T16:42:17,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3e7c74cd51084657a4cbadf00ce766b8, entries=150, sequenceid=420, filesize=12.0 K 2024-11-24T16:42:17,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/3323dd4f201f4f068275bedd183be29b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3323dd4f201f4f068275bedd183be29b 2024-11-24T16:42:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3323dd4f201f4f068275bedd183be29b, entries=150, sequenceid=420, filesize=12.0 K 2024-11-24T16:42:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 3abd8569b30f1874fc795b3a06786690 in 190ms, sequenceid=420, compaction requested=true 2024-11-24T16:42:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:17,180 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-24T16:42:17,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T16:42:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,181 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T16:42:17,181 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T16:42:17,181 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. because compaction request was cancelled 2024-11-24T16:42:17,181 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:17,181 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,182 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:17,182 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:17,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,182 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,183 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ff1f6e5134034b6fa04cde0b8bf82fab, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/8da1d713bc7245c6a5eee9d0d9827e89, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3323dd4f201f4f068275bedd183be29b] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=36.6 K 2024-11-24T16:42:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,183 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ff1f6e5134034b6fa04cde0b8bf82fab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732466535063 2024-11-24T16:42:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,183 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da1d713bc7245c6a5eee9d0d9827e89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732466536343 2024-11-24T16:42:17,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,184 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3323dd4f201f4f068275bedd183be29b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732466536640 2024-11-24T16:42:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,192 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#373 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,192 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/311d5b5e1ed6410193949fb130ba8d96 is 50, key is test_row_0/C:col10/1732466536640/Put/seqid=0 2024-11-24T16:42:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742266_1442 (size=12983) 2024-11-24T16:42:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,203 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/311d5b5e1ed6410193949fb130ba8d96 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/311d5b5e1ed6410193949fb130ba8d96 2024-11-24T16:42:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,207 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into 311d5b5e1ed6410193949fb130ba8d96(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:17,208 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:17,208 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=13, startTime=1732466537180; duration=0sec 2024-11-24T16:42:17,208 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,208 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:17,208 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T16:42:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,209 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T16:42:17,209 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T16:42:17,209 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. because compaction request was cancelled 2024-11-24T16:42:17,209 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:17,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:17,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124836845202ff846f39c3d2ce95aba66d5_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466537363/Put/seqid=0 2024-11-24T16:42:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742267_1443 (size=14994) 2024-11-24T16:42:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,395 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124836845202ff846f39c3d2ce95aba66d5_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124836845202ff846f39c3d2ce95aba66d5_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/e489193e3f1b400799f9763c61c9fb2a, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/e489193e3f1b400799f9763c61c9fb2a is 175, key is test_row_0/A:col10/1732466537363/Put/seqid=0 2024-11-24T16:42:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742268_1444 (size=39945) 2024-11-24T16:42:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,404 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=432, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/e489193e3f1b400799f9763c61c9fb2a 2024-11-24T16:42:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/42d4612190b94ffc8e71f6d3d063fbd9 is 50, key is test_row_0/B:col10/1732466537363/Put/seqid=0 2024-11-24T16:42:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742269_1445 (size=9857) 2024-11-24T16:42:17,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/42d4612190b94ffc8e71f6d3d063fbd9 2024-11-24T16:42:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/f66da1a172b7409789f717f527e70094 is 50, key is test_row_0/C:col10/1732466537363/Put/seqid=0 2024-11-24T16:42:17,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742270_1446 (size=9857) 2024-11-24T16:42:17,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/f66da1a172b7409789f717f527e70094 2024-11-24T16:42:17,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/e489193e3f1b400799f9763c61c9fb2a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/e489193e3f1b400799f9763c61c9fb2a 2024-11-24T16:42:17,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/e489193e3f1b400799f9763c61c9fb2a, entries=200, sequenceid=432, filesize=39.0 K 2024-11-24T16:42:17,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/42d4612190b94ffc8e71f6d3d063fbd9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/42d4612190b94ffc8e71f6d3d063fbd9 2024-11-24T16:42:17,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/42d4612190b94ffc8e71f6d3d063fbd9, entries=100, sequenceid=432, filesize=9.6 K 2024-11-24T16:42:17,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/f66da1a172b7409789f717f527e70094 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/f66da1a172b7409789f717f527e70094 2024-11-24T16:42:17,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/f66da1a172b7409789f717f527e70094, entries=100, sequenceid=432, filesize=9.6 K 2024-11-24T16:42:17,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=114.05 KB/116790 for 3abd8569b30f1874fc795b3a06786690 in 78ms, sequenceid=432, compaction requested=true 2024-11-24T16:42:17,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:17,458 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-24T16:42:17,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:17,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:17,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:17,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:17,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:17,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T16:42:17,460 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T16:42:17,460 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T16:42:17,460 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. because compaction request was cancelled 2024-11-24T16:42:17,460 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:17,460 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:17,460 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:17,460 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:17,461 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:17,461 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/c15947b4b2654a45812b8f2447b8d4f4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3e7c74cd51084657a4cbadf00ce766b8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/42d4612190b94ffc8e71f6d3d063fbd9] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=34.3 K 2024-11-24T16:42:17,461 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c15947b4b2654a45812b8f2447b8d4f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732466536343 2024-11-24T16:42:17,461 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7c74cd51084657a4cbadf00ce766b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732466536640 2024-11-24T16:42:17,461 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 42d4612190b94ffc8e71f6d3d063fbd9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466537363 2024-11-24T16:42:17,468 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:17,468 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/5b49bf023fa14f1b85826474137d5c28 is 50, key is test_row_0/B:col10/1732466537363/Put/seqid=0 2024-11-24T16:42:17,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466537431, max timestamp=1732466537465Num uniques 859; 2024-11-24T16:42:17,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:42:17,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466537431, max timestamp=1732466537465Num uniques 859; 2024-11-24T16:42:17,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=102.87 KB heapSize=301.95 KB 2024-11-24T16:42:17,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:42:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:17,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:17,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742271_1447 (size=13051) 2024-11-24T16:42:17,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a3263671c86f4deab64dca75e40d7770_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466537451/Put/seqid=0 2024-11-24T16:42:17,484 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/5b49bf023fa14f1b85826474137d5c28 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5b49bf023fa14f1b85826474137d5c28 2024-11-24T16:42:17,490 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into 5b49bf023fa14f1b85826474137d5c28(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:17,490 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:17,490 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=13, startTime=1732466537458; duration=0sec 2024-11-24T16:42:17,490 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:17,490 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:17,490 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T16:42:17,490 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T16:42:17,490 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T16:42:17,490 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. because compaction request was cancelled 2024-11-24T16:42:17,491 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:17,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742272_1448 (size=22618) 2024-11-24T16:42:17,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:17,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a3263671c86f4deab64dca75e40d7770_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a3263671c86f4deab64dca75e40d7770_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:17,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/268d601c3eca4bd781ea2711000b3e45, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:17,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/268d601c3eca4bd781ea2711000b3e45 is 175, key is test_row_0/A:col10/1732466537451/Put/seqid=0 2024-11-24T16:42:17,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742273_1449 (size=66023) 2024-11-24T16:42:17,519 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/078ff12fe8484270b8214c0e0cb46a9c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/078ff12fe8484270b8214c0e0cb46a9c 2024-11-24T16:42:17,523 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into 078ff12fe8484270b8214c0e0cb46a9c(size=31.2 K), total size for store is 100.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:17,523 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:17,523 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=13, startTime=1732466536976; duration=0sec 2024-11-24T16:42:17,523 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:17,523 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:17,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466597517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466597519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466597526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466597527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466597644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466597644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466597644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466597645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466597854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466597854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466597854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466597854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:17,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=456, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/268d601c3eca4bd781ea2711000b3e45 2024-11-24T16:42:17,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/12f22d81f8154eae9b18bc23be3c7608 is 50, key is test_row_0/B:col10/1732466537451/Put/seqid=0 2024-11-24T16:42:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742274_1450 (size=12301) 2024-11-24T16:42:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T16:42:18,014 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:18,014 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:18,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-24T16:42:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T16:42:18,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:18,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:18,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:18,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T16:42:18,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T16:42:18,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:18,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466598164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466598164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466598164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466598173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/12f22d81f8154eae9b18bc23be3c7608 2024-11-24T16:42:18,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T16:42:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/877b9e12c6d54c2bad00027a93e41235 is 50, key is test_row_0/C:col10/1732466537451/Put/seqid=0 2024-11-24T16:42:18,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T16:42:18,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742275_1451 (size=12301) 2024-11-24T16:42:18,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T16:42:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T16:42:18,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T16:42:18,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50924 deadline: 1732466598664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,672 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 3abd8569b30f1874fc795b3a06786690 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18362 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466598673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466598683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466598684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466598694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:18,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:18,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/877b9e12c6d54c2bad00027a93e41235 2024-11-24T16:42:18,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/268d601c3eca4bd781ea2711000b3e45 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/268d601c3eca4bd781ea2711000b3e45 2024-11-24T16:42:18,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/268d601c3eca4bd781ea2711000b3e45, entries=350, sequenceid=456, filesize=64.5 K 2024-11-24T16:42:18,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/12f22d81f8154eae9b18bc23be3c7608 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/12f22d81f8154eae9b18bc23be3c7608 2024-11-24T16:42:18,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/12f22d81f8154eae9b18bc23be3c7608, entries=150, sequenceid=456, filesize=12.0 K 2024-11-24T16:42:18,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/877b9e12c6d54c2bad00027a93e41235 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/877b9e12c6d54c2bad00027a93e41235 2024-11-24T16:42:18,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/877b9e12c6d54c2bad00027a93e41235, entries=150, sequenceid=456, filesize=12.0 K 2024-11-24T16:42:18,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~78.27 KB/80150, heapSize ~223.89 KB/229264, currentSize=100.63 KB/103050 for 3abd8569b30f1874fc795b3a06786690 in 1282ms, sequenceid=456, compaction requested=true 2024-11-24T16:42:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:18,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:18,754 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:18,754 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T16:42:18,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:18,754 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T16:42:18,754 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T16:42:18,754 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. because compaction request was cancelled 2024-11-24T16:42:18,754 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:18,754 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:18,755 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 169126 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:18,755 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:18,755 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,755 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/078ff12fe8484270b8214c0e0cb46a9c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4c7c7ff97db04507b6f04d6218e02f13, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/e489193e3f1b400799f9763c61c9fb2a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/268d601c3eca4bd781ea2711000b3e45] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=165.2 K 2024-11-24T16:42:18,755 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,755 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:18,755 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/078ff12fe8484270b8214c0e0cb46a9c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4c7c7ff97db04507b6f04d6218e02f13, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/e489193e3f1b400799f9763c61c9fb2a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/268d601c3eca4bd781ea2711000b3e45] 2024-11-24T16:42:18,755 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:18,755 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,755 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/311d5b5e1ed6410193949fb130ba8d96, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/f66da1a172b7409789f717f527e70094, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/877b9e12c6d54c2bad00027a93e41235] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=34.3 K 2024-11-24T16:42:18,756 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 078ff12fe8484270b8214c0e0cb46a9c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1732466536343 2024-11-24T16:42:18,756 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 311d5b5e1ed6410193949fb130ba8d96, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732466536640 2024-11-24T16:42:18,756 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f66da1a172b7409789f717f527e70094, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466537363 2024-11-24T16:42:18,756 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7c7ff97db04507b6f04d6218e02f13, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1732466536640 2024-11-24T16:42:18,756 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 877b9e12c6d54c2bad00027a93e41235, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732466537451 2024-11-24T16:42:18,756 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e489193e3f1b400799f9763c61c9fb2a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466537009 2024-11-24T16:42:18,756 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268d601c3eca4bd781ea2711000b3e45, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732466537431 2024-11-24T16:42:18,773 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:18,774 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/1f710b31bdac451e8ef4a4619a7325b0 is 50, key is test_row_0/C:col10/1732466537451/Put/seqid=0 2024-11-24T16:42:18,777 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:18,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T16:42:18,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:18,784 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T16:42:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:18,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742276_1452 (size=13085) 2024-11-24T16:42:18,794 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246a29d2c6029b46cbb31e5e7791c0134c_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:18,797 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246a29d2c6029b46cbb31e5e7791c0134c_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:18,797 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246a29d2c6029b46cbb31e5e7791c0134c_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:18,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112469db14c87b6e4ac4ab0f327a06fc3c86_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466537507/Put/seqid=0 2024-11-24T16:42:18,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742277_1453 (size=4469) 2024-11-24T16:42:18,811 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#382 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:18,812 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/5a0002087dad46bdb7f2f89f00fa44aa is 175, key is test_row_0/A:col10/1732466537451/Put/seqid=0 2024-11-24T16:42:18,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742278_1454 (size=12454) 2024-11-24T16:42:18,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742279_1455 (size=32039) 2024-11-24T16:42:18,828 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/5a0002087dad46bdb7f2f89f00fa44aa as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5a0002087dad46bdb7f2f89f00fa44aa 2024-11-24T16:42:18,832 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into 5a0002087dad46bdb7f2f89f00fa44aa(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:18,832 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:18,832 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=12, startTime=1732466538753; duration=0sec 2024-11-24T16:42:18,832 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:18,832 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T16:42:19,202 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/1f710b31bdac451e8ef4a4619a7325b0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1f710b31bdac451e8ef4a4619a7325b0 2024-11-24T16:42:19,206 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into 1f710b31bdac451e8ef4a4619a7325b0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:19,206 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:19,206 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=13, startTime=1732466538754; duration=0sec 2024-11-24T16:42:19,206 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:19,206 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:19,223 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112469db14c87b6e4ac4ab0f327a06fc3c86_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112469db14c87b6e4ac4ab0f327a06fc3c86_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/15eefed6a350461790aa7cc6ec92efeb, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:19,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/15eefed6a350461790aa7cc6ec92efeb is 175, key is test_row_0/A:col10/1732466537507/Put/seqid=0 2024-11-24T16:42:19,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742280_1456 (size=31255) 2024-11-24T16:42:19,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=476, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/15eefed6a350461790aa7cc6ec92efeb 2024-11-24T16:42:19,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/7f3aae14114745d09f0de7ad12f76b23 is 50, key is test_row_0/B:col10/1732466537507/Put/seqid=0 2024-11-24T16:42:19,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742281_1457 (size=12301) 2024-11-24T16:42:19,639 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/7f3aae14114745d09f0de7ad12f76b23 2024-11-24T16:42:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/098acdf9e72b4b51bd55e9a7e9cd0db5 is 50, key is test_row_0/C:col10/1732466537507/Put/seqid=0 2024-11-24T16:42:19,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742282_1458 (size=12301) 2024-11-24T16:42:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:19,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:19,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466599726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466599727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466599738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466599738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466599844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466599844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466599844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466599844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,049 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/098acdf9e72b4b51bd55e9a7e9cd0db5 2024-11-24T16:42:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/15eefed6a350461790aa7cc6ec92efeb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/15eefed6a350461790aa7cc6ec92efeb 2024-11-24T16:42:20,056 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/15eefed6a350461790aa7cc6ec92efeb, entries=150, sequenceid=476, filesize=30.5 K 2024-11-24T16:42:20,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/7f3aae14114745d09f0de7ad12f76b23 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7f3aae14114745d09f0de7ad12f76b23 2024-11-24T16:42:20,060 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7f3aae14114745d09f0de7ad12f76b23, entries=150, sequenceid=476, filesize=12.0 K 2024-11-24T16:42:20,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/098acdf9e72b4b51bd55e9a7e9cd0db5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/098acdf9e72b4b51bd55e9a7e9cd0db5 2024-11-24T16:42:20,064 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/098acdf9e72b4b51bd55e9a7e9cd0db5, entries=150, sequenceid=476, filesize=12.0 K 2024-11-24T16:42:20,065 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 3abd8569b30f1874fc795b3a06786690 in 1281ms, sequenceid=476, compaction requested=true 2024-11-24T16:42:20,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:20,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-24T16:42:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-24T16:42:20,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-24T16:42:20,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0490 sec 2024-11-24T16:42:20,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.0530 sec 2024-11-24T16:42:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:20,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T16:42:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:20,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cd16f4a0e10847a9aff2d67cdd8ad608_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466539723/Put/seqid=0 2024-11-24T16:42:20,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742283_1459 (size=14994) 2024-11-24T16:42:20,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466600099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466600100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466600105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466600106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T16:42:20,164 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:20,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-24T16:42:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T16:42:20,167 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:20,167 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:20,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:20,208 INFO [Thread-1636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1636. 2024-11-24T16:42:20,208 DEBUG [Thread-1636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:20,208 DEBUG [Thread-1636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,208 DEBUG [Thread-1636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:20,215 INFO [Thread-1634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1634. 2024-11-24T16:42:20,215 DEBUG [Thread-1634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:20,215 DEBUG [Thread-1634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,215 DEBUG [Thread-1634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:20,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466600213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466600214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,221 INFO [Thread-1632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1632. 2024-11-24T16:42:20,221 DEBUG [Thread-1632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:20,221 DEBUG [Thread-1632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,221 DEBUG [Thread-1632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,221 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:20,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,224 INFO [Thread-1628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1628. 2024-11-24T16:42:20,224 DEBUG [Thread-1628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466600224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,224 DEBUG [Thread-1628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,224 DEBUG [Thread-1628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,227 INFO [Thread-1630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1630. 2024-11-24T16:42:20,227 DEBUG [Thread-1630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:20,227 DEBUG [Thread-1630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,227 DEBUG [Thread-1630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:20,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:20,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466600234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T16:42:20,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:20,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466600423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466600423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466600434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466600444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,483 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T16:42:20,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cd16f4a0e10847a9aff2d67cdd8ad608_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd16f4a0e10847a9aff2d67cdd8ad608_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:20,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8c45e74c3dad434db46d391d91db52e1, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:20,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8c45e74c3dad434db46d391d91db52e1 is 175, key is test_row_0/A:col10/1732466539723/Put/seqid=0 2024-11-24T16:42:20,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742284_1460 (size=39949) 2024-11-24T16:42:20,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466600734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466600734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466600743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466600753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:20,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:20,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T16:42:20,892 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=494, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8c45e74c3dad434db46d391d91db52e1 2024-11-24T16:42:20,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d9fa2a4673144bd8b8ca3309001c3f1c is 50, key is test_row_0/B:col10/1732466539723/Put/seqid=0 2024-11-24T16:42:20,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742285_1461 (size=12301) 2024-11-24T16:42:20,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:20,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:21,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51006 deadline: 1732466601244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732466601244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:21,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50966 deadline: 1732466601254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:21,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732466601263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:42:21,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d9fa2a4673144bd8b8ca3309001c3f1c 2024-11-24T16:42:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T16:42:21,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/a50c41574f9e48cf9a84006aabf3c81f is 50, key is test_row_0/C:col10/1732466539723/Put/seqid=0 2024-11-24T16:42:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742286_1462 (size=12301) 2024-11-24T16:42:21,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:21,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/a50c41574f9e48cf9a84006aabf3c81f 2024-11-24T16:42:21,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/8c45e74c3dad434db46d391d91db52e1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8c45e74c3dad434db46d391d91db52e1 2024-11-24T16:42:21,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8c45e74c3dad434db46d391d91db52e1, entries=200, sequenceid=494, filesize=39.0 K 2024-11-24T16:42:21,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/d9fa2a4673144bd8b8ca3309001c3f1c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d9fa2a4673144bd8b8ca3309001c3f1c 2024-11-24T16:42:21,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d9fa2a4673144bd8b8ca3309001c3f1c, entries=150, sequenceid=494, filesize=12.0 K 2024-11-24T16:42:21,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/a50c41574f9e48cf9a84006aabf3c81f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/a50c41574f9e48cf9a84006aabf3c81f 2024-11-24T16:42:21,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/a50c41574f9e48cf9a84006aabf3c81f, entries=150, sequenceid=494, filesize=12.0 K 2024-11-24T16:42:21,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 3abd8569b30f1874fc795b3a06786690 in 1651ms, sequenceid=494, compaction requested=true 2024-11-24T16:42:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3abd8569b30f1874fc795b3a06786690:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:21,723 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:21,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:21,723 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:21,724 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:21,724 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/A is initiating minor compaction (all files) 2024-11-24T16:42:21,724 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:21,724 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/A in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,724 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/B is initiating minor compaction (all files) 2024-11-24T16:42:21,724 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/B in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,724 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5a0002087dad46bdb7f2f89f00fa44aa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/15eefed6a350461790aa7cc6ec92efeb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8c45e74c3dad434db46d391d91db52e1] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=100.8 K 2024-11-24T16:42:21,724 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,724 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5b49bf023fa14f1b85826474137d5c28, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/12f22d81f8154eae9b18bc23be3c7608, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7f3aae14114745d09f0de7ad12f76b23, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d9fa2a4673144bd8b8ca3309001c3f1c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=48.8 K 2024-11-24T16:42:21,724 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5a0002087dad46bdb7f2f89f00fa44aa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/15eefed6a350461790aa7cc6ec92efeb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8c45e74c3dad434db46d391d91db52e1] 2024-11-24T16:42:21,725 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a0002087dad46bdb7f2f89f00fa44aa, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732466537451 2024-11-24T16:42:21,725 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b49bf023fa14f1b85826474137d5c28, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1732466536985 2024-11-24T16:42:21,725 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15eefed6a350461790aa7cc6ec92efeb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732466537507 2024-11-24T16:42:21,725 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f22d81f8154eae9b18bc23be3c7608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732466537451 2024-11-24T16:42:21,725 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c45e74c3dad434db46d391d91db52e1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732466539723 2024-11-24T16:42:21,725 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f3aae14114745d09f0de7ad12f76b23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732466537507 2024-11-24T16:42:21,725 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d9fa2a4673144bd8b8ca3309001c3f1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732466539723 2024-11-24T16:42:21,730 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:21,732 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#B#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:21,732 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3a990e39ff6c44f494c72775720d3350 is 50, key is test_row_0/B:col10/1732466539723/Put/seqid=0 2024-11-24T16:42:21,733 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112404353f434d894d4c91a07d6090b230f2_3abd8569b30f1874fc795b3a06786690 store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:21,737 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112404353f434d894d4c91a07d6090b230f2_3abd8569b30f1874fc795b3a06786690, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:21,737 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112404353f434d894d4c91a07d6090b230f2_3abd8569b30f1874fc795b3a06786690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:21,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742287_1463 (size=13187) 2024-11-24T16:42:21,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742288_1464 (size=4469) 2024-11-24T16:42:21,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-24T16:42:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:21,846 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T16:42:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:21,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245d74880cf8214de9a7db4460a07fb7da_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466540105/Put/seqid=0 2024-11-24T16:42:21,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742289_1465 (size=12454) 2024-11-24T16:42:22,141 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#A#compaction#389 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:22,142 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/5c9a11fa44f94cf1b973cc7c0436a647 is 175, key is test_row_0/A:col10/1732466539723/Put/seqid=0 2024-11-24T16:42:22,143 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3a990e39ff6c44f494c72775720d3350 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3a990e39ff6c44f494c72775720d3350 2024-11-24T16:42:22,146 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/B of 3abd8569b30f1874fc795b3a06786690 into 3a990e39ff6c44f494c72775720d3350(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:22,147 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:22,147 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/B, priority=12, startTime=1732466541723; duration=0sec 2024-11-24T16:42:22,147 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:22,147 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:B 2024-11-24T16:42:22,147 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:22,147 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:22,147 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 3abd8569b30f1874fc795b3a06786690/C is initiating minor compaction (all files) 2024-11-24T16:42:22,147 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3abd8569b30f1874fc795b3a06786690/C in TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:22,147 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1f710b31bdac451e8ef4a4619a7325b0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/098acdf9e72b4b51bd55e9a7e9cd0db5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/a50c41574f9e48cf9a84006aabf3c81f] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp, totalSize=36.8 K 2024-11-24T16:42:22,149 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f710b31bdac451e8ef4a4619a7325b0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1732466537451 2024-11-24T16:42:22,149 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 098acdf9e72b4b51bd55e9a7e9cd0db5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732466537507 2024-11-24T16:42:22,149 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting a50c41574f9e48cf9a84006aabf3c81f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732466539723 2024-11-24T16:42:22,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742290_1466 (size=32141) 2024-11-24T16:42:22,156 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3abd8569b30f1874fc795b3a06786690#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:22,157 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/38082204a0c74fc28b2f41e08e300d92 is 50, key is test_row_0/C:col10/1732466539723/Put/seqid=0 2024-11-24T16:42:22,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742291_1467 (size=13187) 2024-11-24T16:42:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:22,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. as already flushing 2024-11-24T16:42:22,255 INFO [Thread-1625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1625. 2024-11-24T16:42:22,255 DEBUG [Thread-1625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:22,255 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,255 DEBUG [Thread-1625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:22,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:22,258 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245d74880cf8214de9a7db4460a07fb7da_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d74880cf8214de9a7db4460a07fb7da_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:22,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/ab00d232b4524312962777639475350e, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:22,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/ab00d232b4524312962777639475350e is 175, key is test_row_0/A:col10/1732466540105/Put/seqid=0 2024-11-24T16:42:22,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742292_1468 (size=31255) 2024-11-24T16:42:22,265 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-11-24T16:42:22,265 INFO [Thread-1623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1623. 2024-11-24T16:42:22,265 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:22,265 DEBUG [Thread-1623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:22,265 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,265 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,266 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,266 DEBUG [Thread-1623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:22,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:22,284 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-11-24T16:42:22,284 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:22,284 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,285 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:22,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T16:42:22,554 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/5c9a11fa44f94cf1b973cc7c0436a647 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5c9a11fa44f94cf1b973cc7c0436a647 2024-11-24T16:42:22,557 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/A of 3abd8569b30f1874fc795b3a06786690 into 5c9a11fa44f94cf1b973cc7c0436a647(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:22,557 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:22,557 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/A, priority=13, startTime=1732466541723; duration=0sec 2024-11-24T16:42:22,557 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:22,558 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:A 2024-11-24T16:42:22,567 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/38082204a0c74fc28b2f41e08e300d92 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/38082204a0c74fc28b2f41e08e300d92 2024-11-24T16:42:22,571 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3abd8569b30f1874fc795b3a06786690/C of 3abd8569b30f1874fc795b3a06786690 into 38082204a0c74fc28b2f41e08e300d92(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:22,571 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:22,571 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., storeName=3abd8569b30f1874fc795b3a06786690/C, priority=13, startTime=1732466541723; duration=0sec 2024-11-24T16:42:22,571 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:22,571 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3abd8569b30f1874fc795b3a06786690:C 2024-11-24T16:42:22,663 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=510, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/ab00d232b4524312962777639475350e 2024-11-24T16:42:22,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/2f91592d1b6142418bf477fa9f665d73 is 50, key is test_row_0/B:col10/1732466540105/Put/seqid=0 2024-11-24T16:42:22,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742293_1469 (size=12301) 2024-11-24T16:42:23,072 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/2f91592d1b6142418bf477fa9f665d73 2024-11-24T16:42:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/fe9b9ccfa0a74db3801c64783630f1b6 is 50, key is test_row_0/C:col10/1732466540105/Put/seqid=0 2024-11-24T16:42:23,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742294_1470 (size=12301) 2024-11-24T16:42:23,481 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/fe9b9ccfa0a74db3801c64783630f1b6 2024-11-24T16:42:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/ab00d232b4524312962777639475350e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/ab00d232b4524312962777639475350e 2024-11-24T16:42:23,487 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/ab00d232b4524312962777639475350e, entries=150, sequenceid=510, filesize=30.5 K 2024-11-24T16:42:23,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/2f91592d1b6142418bf477fa9f665d73 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2f91592d1b6142418bf477fa9f665d73 2024-11-24T16:42:23,490 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2f91592d1b6142418bf477fa9f665d73, entries=150, sequenceid=510, filesize=12.0 K 2024-11-24T16:42:23,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/fe9b9ccfa0a74db3801c64783630f1b6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/fe9b9ccfa0a74db3801c64783630f1b6 2024-11-24T16:42:23,493 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/fe9b9ccfa0a74db3801c64783630f1b6, entries=150, sequenceid=510, filesize=12.0 K 2024-11-24T16:42:23,494 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 3abd8569b30f1874fc795b3a06786690 in 1648ms, sequenceid=510, compaction requested=false 2024-11-24T16:42:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 3abd8569b30f1874fc795b3a06786690: 2024-11-24T16:42:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-24T16:42:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-24T16:42:23,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-24T16:42:23,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3280 sec 2024-11-24T16:42:23,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.3320 sec 2024-11-24T16:42:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T16:42:24,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:28,715 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-11-24T16:42:28,715 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:28,715 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:28,715 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:28,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 826 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2477 rows 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 829 2024-11-24T16:42:28,715 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2483 rows 2024-11-24T16:42:28,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 828 2024-11-24T16:42:28,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2481 rows 2024-11-24T16:42:28,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 827 2024-11-24T16:42:28,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2481 rows 2024-11-24T16:42:28,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 829 2024-11-24T16:42:28,716 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2481 rows 2024-11-24T16:42:28,716 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:42:28,716 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:42:28,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:28,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:28,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:28,717 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T16:42:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T16:42:28,719 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466548719"}]},"ts":"1732466548719"} 2024-11-24T16:42:28,720 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T16:42:28,720 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T16:42:28,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:42:28,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, UNASSIGN}] 2024-11-24T16:42:28,723 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, UNASSIGN 2024-11-24T16:42:28,723 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:42:28,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, UNASSIGN because future has completed 2024-11-24T16:42:28,724 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:42:28,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:42:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T16:42:28,876 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:28,876 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:42:28,876 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 3abd8569b30f1874fc795b3a06786690, disabling compactions & flushes 2024-11-24T16:42:28,876 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:28,876 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:28,876 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. after waiting 0 ms 2024-11-24T16:42:28,876 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:28,877 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 3abd8569b30f1874fc795b3a06786690 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T16:42:28,877 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=A 2024-11-24T16:42:28,877 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:28,877 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=B 2024-11-24T16:42:28,877 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:28,877 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3abd8569b30f1874fc795b3a06786690, store=C 2024-11-24T16:42:28,877 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:28,882 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124789cf9ecade24a8f868d116d011e3ffd_3abd8569b30f1874fc795b3a06786690 is 50, key is test_row_0/A:col10/1732466542264/Put/seqid=0 2024-11-24T16:42:28,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742295_1471 (size=12454) 2024-11-24T16:42:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T16:42:29,286 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:29,289 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124789cf9ecade24a8f868d116d011e3ffd_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124789cf9ecade24a8f868d116d011e3ffd_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:29,289 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/956696d9641b4000a144f68cef5459b6, store: [table=TestAcidGuarantees family=A region=3abd8569b30f1874fc795b3a06786690] 2024-11-24T16:42:29,290 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/956696d9641b4000a144f68cef5459b6 is 175, key is test_row_0/A:col10/1732466542264/Put/seqid=0 2024-11-24T16:42:29,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742296_1472 (size=31255) 2024-11-24T16:42:29,293 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=521, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/956696d9641b4000a144f68cef5459b6 2024-11-24T16:42:29,298 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3dc1d94751894752b864d910a50bdbf8 is 50, key is test_row_0/B:col10/1732466542264/Put/seqid=0 2024-11-24T16:42:29,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742297_1473 (size=12301) 2024-11-24T16:42:29,335 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T16:42:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T16:42:29,704 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3dc1d94751894752b864d910a50bdbf8 2024-11-24T16:42:29,709 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ba23caa049504b3eb8c5ad110eebcc44 is 50, key is test_row_0/C:col10/1732466542264/Put/seqid=0 2024-11-24T16:42:29,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742298_1474 (size=12301) 2024-11-24T16:42:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T16:42:30,113 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ba23caa049504b3eb8c5ad110eebcc44 2024-11-24T16:42:30,116 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/A/956696d9641b4000a144f68cef5459b6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/956696d9641b4000a144f68cef5459b6 2024-11-24T16:42:30,118 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/956696d9641b4000a144f68cef5459b6, entries=150, sequenceid=521, filesize=30.5 K 2024-11-24T16:42:30,119 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/B/3dc1d94751894752b864d910a50bdbf8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3dc1d94751894752b864d910a50bdbf8 2024-11-24T16:42:30,121 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3dc1d94751894752b864d910a50bdbf8, entries=150, sequenceid=521, filesize=12.0 K 2024-11-24T16:42:30,122 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/.tmp/C/ba23caa049504b3eb8c5ad110eebcc44 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba23caa049504b3eb8c5ad110eebcc44 2024-11-24T16:42:30,124 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba23caa049504b3eb8c5ad110eebcc44, entries=150, sequenceid=521, filesize=12.0 K 2024-11-24T16:42:30,125 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3abd8569b30f1874fc795b3a06786690 in 1249ms, sequenceid=521, compaction requested=true 2024-11-24T16:42:30,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/cca2c61bb50b4f77b85ad07f0af6d259, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9f42a5145c52427d860c7cfd12b2ca92, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/015275c0b9b64ca198d6d5233e1c0535, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/a6c7215a24df47249d4d9c99e45e0e22, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/2fb0514132504dbbac38fac24a79aeff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/7bc55378b0cb4c6abeeb1f15ddd0868e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/b557672f3e91496ea18ee250b860f2da, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4f9988cfbc9740beb25edabb307c7430, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/0f55aab357cf41a9bd4bc4fe1ca9bdda, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/6ca4607ceaa447f6a76d3b69a1f7ca60, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/abe6937b48dc4c9db5b4121b241e1a71, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/07b53b32ee3743eab5475af2ff34842a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/144da2df2c494edab6936975a05302ac, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/12dafb5b0de5499c8efb26440fd93980, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9d2f53a25df44cc9b5eef33ee10e47ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/60fcbf0ccce4428dacaf99ca6bb39f43, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/445ba6af815c4a469537940e00e473d5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/111f1f67c93d434bba50a156d16a2a9b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8d4c100e662f45aea147a2b9ff4ef50e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/078ff12fe8484270b8214c0e0cb46a9c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4c7c7ff97db04507b6f04d6218e02f13, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/e489193e3f1b400799f9763c61c9fb2a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/268d601c3eca4bd781ea2711000b3e45, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5a0002087dad46bdb7f2f89f00fa44aa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/15eefed6a350461790aa7cc6ec92efeb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8c45e74c3dad434db46d391d91db52e1] to archive 2024-11-24T16:42:30,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:42:30,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/cca2c61bb50b4f77b85ad07f0af6d259 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/cca2c61bb50b4f77b85ad07f0af6d259 2024-11-24T16:42:30,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9f42a5145c52427d860c7cfd12b2ca92 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9f42a5145c52427d860c7cfd12b2ca92 2024-11-24T16:42:30,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/015275c0b9b64ca198d6d5233e1c0535 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/015275c0b9b64ca198d6d5233e1c0535 2024-11-24T16:42:30,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/a6c7215a24df47249d4d9c99e45e0e22 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/a6c7215a24df47249d4d9c99e45e0e22 2024-11-24T16:42:30,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/2fb0514132504dbbac38fac24a79aeff to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/2fb0514132504dbbac38fac24a79aeff 2024-11-24T16:42:30,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/7bc55378b0cb4c6abeeb1f15ddd0868e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/7bc55378b0cb4c6abeeb1f15ddd0868e 2024-11-24T16:42:30,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/b557672f3e91496ea18ee250b860f2da to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/b557672f3e91496ea18ee250b860f2da 2024-11-24T16:42:30,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4f9988cfbc9740beb25edabb307c7430 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4f9988cfbc9740beb25edabb307c7430 2024-11-24T16:42:30,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/0f55aab357cf41a9bd4bc4fe1ca9bdda to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/0f55aab357cf41a9bd4bc4fe1ca9bdda 2024-11-24T16:42:30,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/6ca4607ceaa447f6a76d3b69a1f7ca60 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/6ca4607ceaa447f6a76d3b69a1f7ca60 2024-11-24T16:42:30,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/abe6937b48dc4c9db5b4121b241e1a71 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/abe6937b48dc4c9db5b4121b241e1a71 2024-11-24T16:42:30,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/07b53b32ee3743eab5475af2ff34842a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/07b53b32ee3743eab5475af2ff34842a 2024-11-24T16:42:30,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/144da2df2c494edab6936975a05302ac to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/144da2df2c494edab6936975a05302ac 2024-11-24T16:42:30,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/12dafb5b0de5499c8efb26440fd93980 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/12dafb5b0de5499c8efb26440fd93980 2024-11-24T16:42:30,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9d2f53a25df44cc9b5eef33ee10e47ea to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/9d2f53a25df44cc9b5eef33ee10e47ea 2024-11-24T16:42:30,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/60fcbf0ccce4428dacaf99ca6bb39f43 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/60fcbf0ccce4428dacaf99ca6bb39f43 2024-11-24T16:42:30,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/445ba6af815c4a469537940e00e473d5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/445ba6af815c4a469537940e00e473d5 2024-11-24T16:42:30,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/111f1f67c93d434bba50a156d16a2a9b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/111f1f67c93d434bba50a156d16a2a9b 2024-11-24T16:42:30,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8d4c100e662f45aea147a2b9ff4ef50e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8d4c100e662f45aea147a2b9ff4ef50e 2024-11-24T16:42:30,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/078ff12fe8484270b8214c0e0cb46a9c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/078ff12fe8484270b8214c0e0cb46a9c 2024-11-24T16:42:30,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4c7c7ff97db04507b6f04d6218e02f13 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/4c7c7ff97db04507b6f04d6218e02f13 2024-11-24T16:42:30,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/e489193e3f1b400799f9763c61c9fb2a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/e489193e3f1b400799f9763c61c9fb2a 2024-11-24T16:42:30,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/268d601c3eca4bd781ea2711000b3e45 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/268d601c3eca4bd781ea2711000b3e45 2024-11-24T16:42:30,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5a0002087dad46bdb7f2f89f00fa44aa to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5a0002087dad46bdb7f2f89f00fa44aa 2024-11-24T16:42:30,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/15eefed6a350461790aa7cc6ec92efeb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/15eefed6a350461790aa7cc6ec92efeb 2024-11-24T16:42:30,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8c45e74c3dad434db46d391d91db52e1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/8c45e74c3dad434db46d391d91db52e1 2024-11-24T16:42:30,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a3a3dda78e724940ab0d95f97971ebcd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/6f14784ea3c34414a79d0ff5318b5a4c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/9d01815c18e14cc3b2af757f2172e431, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/86640e5b1074454fafe0cad9fae3d263, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/4ce6a8d962d24f91891aa94714f4eebd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2026a024e0774df89f6be28dae835bae, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55492f428bab4a78a72acef957bfd968, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7621b8bc2241434ca8b8bffd12e65e09, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/10b7342b31ae4208be4d426aaaf5fcbf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55f5e672dd404d9cb329385de6ab4a16, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d95f648715ee42909ad0830aad40df0c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/94e0bf18085543be8956aadcf5b3267a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/b3916c80f1554fa8bff3e4be5c20384f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d665d2f27dcd4171bd6cf2d8800b2899, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/bbffc9655fe147d08f1da3104b0421b1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a45a417a49f44c29a4c738e3ffa6c3cf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/30731cfaafd24145874a8cd9e42483de, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/0ca394c92a594511929af3b68350e9da, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/c15947b4b2654a45812b8f2447b8d4f4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5af8eb1b18544ab48100e28c28c31717, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3e7c74cd51084657a4cbadf00ce766b8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5b49bf023fa14f1b85826474137d5c28, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/42d4612190b94ffc8e71f6d3d063fbd9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/12f22d81f8154eae9b18bc23be3c7608, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7f3aae14114745d09f0de7ad12f76b23, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d9fa2a4673144bd8b8ca3309001c3f1c] to archive 2024-11-24T16:42:30,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:42:30,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a3a3dda78e724940ab0d95f97971ebcd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a3a3dda78e724940ab0d95f97971ebcd 2024-11-24T16:42:30,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/6f14784ea3c34414a79d0ff5318b5a4c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/6f14784ea3c34414a79d0ff5318b5a4c 2024-11-24T16:42:30,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/9d01815c18e14cc3b2af757f2172e431 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/9d01815c18e14cc3b2af757f2172e431 2024-11-24T16:42:30,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/86640e5b1074454fafe0cad9fae3d263 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/86640e5b1074454fafe0cad9fae3d263 2024-11-24T16:42:30,152 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/4ce6a8d962d24f91891aa94714f4eebd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/4ce6a8d962d24f91891aa94714f4eebd 2024-11-24T16:42:30,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2026a024e0774df89f6be28dae835bae to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2026a024e0774df89f6be28dae835bae 2024-11-24T16:42:30,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55492f428bab4a78a72acef957bfd968 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55492f428bab4a78a72acef957bfd968 2024-11-24T16:42:30,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7621b8bc2241434ca8b8bffd12e65e09 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7621b8bc2241434ca8b8bffd12e65e09 2024-11-24T16:42:30,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/10b7342b31ae4208be4d426aaaf5fcbf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/10b7342b31ae4208be4d426aaaf5fcbf 2024-11-24T16:42:30,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55f5e672dd404d9cb329385de6ab4a16 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/55f5e672dd404d9cb329385de6ab4a16 2024-11-24T16:42:30,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d95f648715ee42909ad0830aad40df0c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d95f648715ee42909ad0830aad40df0c 2024-11-24T16:42:30,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/94e0bf18085543be8956aadcf5b3267a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/94e0bf18085543be8956aadcf5b3267a 2024-11-24T16:42:30,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/b3916c80f1554fa8bff3e4be5c20384f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/b3916c80f1554fa8bff3e4be5c20384f 2024-11-24T16:42:30,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d665d2f27dcd4171bd6cf2d8800b2899 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d665d2f27dcd4171bd6cf2d8800b2899 2024-11-24T16:42:30,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/bbffc9655fe147d08f1da3104b0421b1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/bbffc9655fe147d08f1da3104b0421b1 2024-11-24T16:42:30,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a45a417a49f44c29a4c738e3ffa6c3cf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/a45a417a49f44c29a4c738e3ffa6c3cf 2024-11-24T16:42:30,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/30731cfaafd24145874a8cd9e42483de to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/30731cfaafd24145874a8cd9e42483de 2024-11-24T16:42:30,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/0ca394c92a594511929af3b68350e9da to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/0ca394c92a594511929af3b68350e9da 2024-11-24T16:42:30,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/c15947b4b2654a45812b8f2447b8d4f4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/c15947b4b2654a45812b8f2447b8d4f4 2024-11-24T16:42:30,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5af8eb1b18544ab48100e28c28c31717 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5af8eb1b18544ab48100e28c28c31717 2024-11-24T16:42:30,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3e7c74cd51084657a4cbadf00ce766b8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3e7c74cd51084657a4cbadf00ce766b8 2024-11-24T16:42:30,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5b49bf023fa14f1b85826474137d5c28 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/5b49bf023fa14f1b85826474137d5c28 2024-11-24T16:42:30,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/42d4612190b94ffc8e71f6d3d063fbd9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/42d4612190b94ffc8e71f6d3d063fbd9 2024-11-24T16:42:30,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/12f22d81f8154eae9b18bc23be3c7608 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/12f22d81f8154eae9b18bc23be3c7608 2024-11-24T16:42:30,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7f3aae14114745d09f0de7ad12f76b23 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/7f3aae14114745d09f0de7ad12f76b23 2024-11-24T16:42:30,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d9fa2a4673144bd8b8ca3309001c3f1c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/d9fa2a4673144bd8b8ca3309001c3f1c 2024-11-24T16:42:30,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/4ec904296eab40fa8d015ea837dda863, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/44a8d38fdb384de1838e4beba59c151f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/93c07a2ef63c4648becc57d7e57f4ffd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d30eed588e11498d9f9caf2bc2fb418d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/577c3a8429344692b297461eb9aefcc1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3dcd2eaea4074906a6c92ab166c59a8c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/eff80df9ecfd4ec4981cccae8166dbec, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3e297de3aab64ae8b37460325655ae95, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/af7195301fe74885a726f54426a49eee, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/6a4c2aeba12f4c2583bef1be7ceba549, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1a48b0c4599249ba98c7b0b4713823da, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d0cd2767f81a4e819d4558f7bac94732, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/445c63e30657406b955413a048a32821, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/7c7f3ba3fee7462c8a63a377f1387afb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/c49379567fd44c55a0ce582533d02d3f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d41df726bb53492dbd3bc102c3a7fa80, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ff1f6e5134034b6fa04cde0b8bf82fab, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba39f2cacc9d4940be7f97eecdbe38ff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/8da1d713bc7245c6a5eee9d0d9827e89, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/311d5b5e1ed6410193949fb130ba8d96, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3323dd4f201f4f068275bedd183be29b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/f66da1a172b7409789f717f527e70094, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1f710b31bdac451e8ef4a4619a7325b0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/877b9e12c6d54c2bad00027a93e41235, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/098acdf9e72b4b51bd55e9a7e9cd0db5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/a50c41574f9e48cf9a84006aabf3c81f] to archive 2024-11-24T16:42:30,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:42:30,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/4ec904296eab40fa8d015ea837dda863 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/4ec904296eab40fa8d015ea837dda863 2024-11-24T16:42:30,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/44a8d38fdb384de1838e4beba59c151f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/44a8d38fdb384de1838e4beba59c151f 2024-11-24T16:42:30,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/93c07a2ef63c4648becc57d7e57f4ffd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/93c07a2ef63c4648becc57d7e57f4ffd 2024-11-24T16:42:30,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d30eed588e11498d9f9caf2bc2fb418d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d30eed588e11498d9f9caf2bc2fb418d 2024-11-24T16:42:30,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/577c3a8429344692b297461eb9aefcc1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/577c3a8429344692b297461eb9aefcc1 2024-11-24T16:42:30,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3dcd2eaea4074906a6c92ab166c59a8c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3dcd2eaea4074906a6c92ab166c59a8c 2024-11-24T16:42:30,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/eff80df9ecfd4ec4981cccae8166dbec to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/eff80df9ecfd4ec4981cccae8166dbec 2024-11-24T16:42:30,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3e297de3aab64ae8b37460325655ae95 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3e297de3aab64ae8b37460325655ae95 2024-11-24T16:42:30,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/af7195301fe74885a726f54426a49eee to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/af7195301fe74885a726f54426a49eee 2024-11-24T16:42:30,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/6a4c2aeba12f4c2583bef1be7ceba549 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/6a4c2aeba12f4c2583bef1be7ceba549 2024-11-24T16:42:30,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1a48b0c4599249ba98c7b0b4713823da to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1a48b0c4599249ba98c7b0b4713823da 2024-11-24T16:42:30,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d0cd2767f81a4e819d4558f7bac94732 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d0cd2767f81a4e819d4558f7bac94732 2024-11-24T16:42:30,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/445c63e30657406b955413a048a32821 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/445c63e30657406b955413a048a32821 2024-11-24T16:42:30,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/7c7f3ba3fee7462c8a63a377f1387afb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/7c7f3ba3fee7462c8a63a377f1387afb 2024-11-24T16:42:30,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/c49379567fd44c55a0ce582533d02d3f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/c49379567fd44c55a0ce582533d02d3f 2024-11-24T16:42:30,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d41df726bb53492dbd3bc102c3a7fa80 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/d41df726bb53492dbd3bc102c3a7fa80 2024-11-24T16:42:30,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ff1f6e5134034b6fa04cde0b8bf82fab to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ff1f6e5134034b6fa04cde0b8bf82fab 2024-11-24T16:42:30,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba39f2cacc9d4940be7f97eecdbe38ff to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba39f2cacc9d4940be7f97eecdbe38ff 2024-11-24T16:42:30,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/8da1d713bc7245c6a5eee9d0d9827e89 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/8da1d713bc7245c6a5eee9d0d9827e89 2024-11-24T16:42:30,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/311d5b5e1ed6410193949fb130ba8d96 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/311d5b5e1ed6410193949fb130ba8d96 2024-11-24T16:42:30,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3323dd4f201f4f068275bedd183be29b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/3323dd4f201f4f068275bedd183be29b 2024-11-24T16:42:30,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/f66da1a172b7409789f717f527e70094 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/f66da1a172b7409789f717f527e70094 2024-11-24T16:42:30,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1f710b31bdac451e8ef4a4619a7325b0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/1f710b31bdac451e8ef4a4619a7325b0 2024-11-24T16:42:30,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/877b9e12c6d54c2bad00027a93e41235 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/877b9e12c6d54c2bad00027a93e41235 2024-11-24T16:42:30,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/098acdf9e72b4b51bd55e9a7e9cd0db5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/098acdf9e72b4b51bd55e9a7e9cd0db5 2024-11-24T16:42:30,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/a50c41574f9e48cf9a84006aabf3c81f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/a50c41574f9e48cf9a84006aabf3c81f 2024-11-24T16:42:30,193 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/recovered.edits/524.seqid, newMaxSeqId=524, maxSeqId=4 2024-11-24T16:42:30,193 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. 2024-11-24T16:42:30,193 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 3abd8569b30f1874fc795b3a06786690: Waiting for close lock at 1732466548876Running coprocessor pre-close hooks at 1732466548876Disabling compacts and flushes for region at 1732466548876Disabling writes for close at 1732466548876Obtaining lock to block concurrent updates at 1732466548877 (+1 ms)Preparing flush snapshotting stores in 3abd8569b30f1874fc795b3a06786690 at 1732466548877Finished memstore snapshotting TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732466548877Flushing stores of TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690. at 1732466548877Flushing 3abd8569b30f1874fc795b3a06786690/A: creating writer at 1732466548877Flushing mob file 3abd8569b30f1874fc795b3a06786690/A: appending metadata at 1732466548882 (+5 ms)Flushing mob file 3abd8569b30f1874fc795b3a06786690/A: closing flushed file at 1732466548882Flushing 3abd8569b30f1874fc795b3a06786690/A: appending metadata at 1732466549290 (+408 ms)Flushing 3abd8569b30f1874fc795b3a06786690/A: closing flushed file at 1732466549290Flushing 3abd8569b30f1874fc795b3a06786690/B: creating writer at 1732466549296 (+6 ms)Flushing 3abd8569b30f1874fc795b3a06786690/B: appending metadata at 1732466549298 (+2 ms)Flushing 3abd8569b30f1874fc795b3a06786690/B: closing flushed file at 1732466549298Flushing 3abd8569b30f1874fc795b3a06786690/C: creating writer at 1732466549707 (+409 ms)Flushing 3abd8569b30f1874fc795b3a06786690/C: appending metadata at 1732466549709 (+2 ms)Flushing 3abd8569b30f1874fc795b3a06786690/C: closing flushed file at 1732466549709Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@725863bc: reopening flushed file at 1732466550115 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24cd83c: reopening flushed file at 1732466550118 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73f29f41: reopening flushed file at 1732466550121 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 3abd8569b30f1874fc795b3a06786690 in 1249ms, sequenceid=521, compaction requested=true at 1732466550125 (+4 ms)Writing region close event to WAL at 1732466550190 (+65 ms)Running coprocessor post-close hooks at 1732466550193 (+3 ms)Closed at 1732466550193 2024-11-24T16:42:30,195 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,195 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=3abd8569b30f1874fc795b3a06786690, regionState=CLOSED 2024-11-24T16:42:30,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:42:30,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-24T16:42:30,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 3abd8569b30f1874fc795b3a06786690, server=e335232f7491,33607,1732466431346 in 1.4730 sec 2024-11-24T16:42:30,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-24T16:42:30,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3abd8569b30f1874fc795b3a06786690, UNASSIGN in 1.4760 sec 2024-11-24T16:42:30,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-24T16:42:30,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4790 sec 2024-11-24T16:42:30,203 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466550202"}]},"ts":"1732466550202"} 2024-11-24T16:42:30,204 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T16:42:30,204 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T16:42:30,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4870 sec 2024-11-24T16:42:30,789 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T16:42:30,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-24T16:42:30,864 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:30,864 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T16:42:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:30,866 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T16:42:30,867 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:30,868 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,870 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/recovered.edits] 2024-11-24T16:42:30,872 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5c9a11fa44f94cf1b973cc7c0436a647 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/5c9a11fa44f94cf1b973cc7c0436a647 2024-11-24T16:42:30,873 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/956696d9641b4000a144f68cef5459b6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/956696d9641b4000a144f68cef5459b6 2024-11-24T16:42:30,874 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/ab00d232b4524312962777639475350e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/A/ab00d232b4524312962777639475350e 2024-11-24T16:42:30,875 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2f91592d1b6142418bf477fa9f665d73 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/2f91592d1b6142418bf477fa9f665d73 2024-11-24T16:42:30,876 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3a990e39ff6c44f494c72775720d3350 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3a990e39ff6c44f494c72775720d3350 2024-11-24T16:42:30,877 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3dc1d94751894752b864d910a50bdbf8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/B/3dc1d94751894752b864d910a50bdbf8 2024-11-24T16:42:30,878 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/38082204a0c74fc28b2f41e08e300d92 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/38082204a0c74fc28b2f41e08e300d92 2024-11-24T16:42:30,879 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba23caa049504b3eb8c5ad110eebcc44 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/ba23caa049504b3eb8c5ad110eebcc44 2024-11-24T16:42:30,880 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/fe9b9ccfa0a74db3801c64783630f1b6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/C/fe9b9ccfa0a74db3801c64783630f1b6 2024-11-24T16:42:30,882 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/recovered.edits/524.seqid to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690/recovered.edits/524.seqid 2024-11-24T16:42:30,882 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,882 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T16:42:30,883 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T16:42:30,883 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T16:42:30,885 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240daa8eb8f97448d2b9da7c830ffa7944_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240daa8eb8f97448d2b9da7c830ffa7944_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,886 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240ed62ad83eb9463cb707bf861fda1479_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240ed62ad83eb9463cb707bf861fda1479_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,886 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124146895b6e99042f79e96d526b41cca41_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124146895b6e99042f79e96d526b41cca41_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,887 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112428af3b69e7ca4eefaf62cbfbaaab9544_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112428af3b69e7ca4eefaf62cbfbaaab9544_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,888 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112431064545bb3c4bff99b8f7f43fc659a2_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112431064545bb3c4bff99b8f7f43fc659a2_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,888 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243d320f5846b144199a4e8d66afc66ab4_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243d320f5846b144199a4e8d66afc66ab4_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,889 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112447b4aca1e2ab416784846a3e2e2d3e97_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112447b4aca1e2ab416784846a3e2e2d3e97_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,890 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456a1938df70d48f9bdbe9326b27927f2_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456a1938df70d48f9bdbe9326b27927f2_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,890 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d74880cf8214de9a7db4460a07fb7da_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d74880cf8214de9a7db4460a07fb7da_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,891 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d9c7ccb36884328ba0cdcc8a4bbc978_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245d9c7ccb36884328ba0cdcc8a4bbc978_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,892 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112469db14c87b6e4ac4ab0f327a06fc3c86_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112469db14c87b6e4ac4ab0f327a06fc3c86_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,892 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124789cf9ecade24a8f868d116d011e3ffd_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124789cf9ecade24a8f868d116d011e3ffd_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,893 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247e7bbeeffbcc43df94e62a6b814253c7_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411247e7bbeeffbcc43df94e62a6b814253c7_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,894 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124836845202ff846f39c3d2ce95aba66d5_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124836845202ff846f39c3d2ce95aba66d5_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,895 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248aeb96497e0d48deb4c8cd91baa8f4b0_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248aeb96497e0d48deb4c8cd91baa8f4b0_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,895 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a3263671c86f4deab64dca75e40d7770_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a3263671c86f4deab64dca75e40d7770_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,896 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b92c4017d4f04e60bd2d892b263f61ef_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b92c4017d4f04e60bd2d892b263f61ef_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,897 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9d9219ee950438087d1ed19b8ccf173_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9d9219ee950438087d1ed19b8ccf173_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,898 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c7134305170948f6880ed8d65221489d_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c7134305170948f6880ed8d65221489d_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,898 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c734325071b541988638c03e8b68f144_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c734325071b541988638c03e8b68f144_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,899 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd16f4a0e10847a9aff2d67cdd8ad608_3abd8569b30f1874fc795b3a06786690 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd16f4a0e10847a9aff2d67cdd8ad608_3abd8569b30f1874fc795b3a06786690 2024-11-24T16:42:30,899 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T16:42:30,901 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:30,903 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T16:42:30,904 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T16:42:30,905 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:30,905 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T16:42:30,905 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732466550905"}]},"ts":"9223372036854775807"} 2024-11-24T16:42:30,907 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T16:42:30,907 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3abd8569b30f1874fc795b3a06786690, NAME => 'TestAcidGuarantees,,1732466517103.3abd8569b30f1874fc795b3a06786690.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T16:42:30,907 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T16:42:30,907 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732466550907"}]},"ts":"9223372036854775807"} 2024-11-24T16:42:30,908 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T16:42:30,909 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:30,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-24T16:42:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-24T16:42:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T16:42:30,974 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:30,983 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=245 (was 249), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=461 (was 474), ProcessCount=11 (was 11), AvailableMemoryMB=5842 (was 6387) 2024-11-24T16:42:30,991 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=245, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=461, ProcessCount=11, AvailableMemoryMB=5841 2024-11-24T16:42:30,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:42:30,992 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:42:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T16:42:30,993 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:42:30,994 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:30,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-24T16:42:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-24T16:42:30,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:42:30,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742299_1475 (size=1061) 2024-11-24T16:42:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-24T16:42:31,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-24T16:42:31,400 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:42:31,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742300_1476 (size=53) 2024-11-24T16:42:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-24T16:42:31,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:42:31,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 10af3cc4e904fab18c276c01940de289, disabling compactions & flushes 2024-11-24T16:42:31,805 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:31,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:31,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. after waiting 0 ms 2024-11-24T16:42:31,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:31,805 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:31,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 10af3cc4e904fab18c276c01940de289: Waiting for close lock at 1732466551805Disabling compacts and flushes for region at 1732466551805Disabling writes for close at 1732466551805Writing region close event to WAL at 1732466551805Closed at 1732466551805 2024-11-24T16:42:31,806 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:42:31,806 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732466551806"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466551806"}]},"ts":"1732466551806"} 2024-11-24T16:42:31,808 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:42:31,808 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:42:31,808 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466551808"}]},"ts":"1732466551808"} 2024-11-24T16:42:31,809 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T16:42:31,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, ASSIGN}] 2024-11-24T16:42:31,811 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, ASSIGN 2024-11-24T16:42:31,811 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, ASSIGN; state=OFFLINE, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=false 2024-11-24T16:42:31,962 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=10af3cc4e904fab18c276c01940de289, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:42:31,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, ASSIGN because future has completed 2024-11-24T16:42:31,963 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:42:32,117 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:32,117 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:42:32,118 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,118 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:42:32,118 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,118 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,119 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,120 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:42:32,120 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10af3cc4e904fab18c276c01940de289 columnFamilyName A 2024-11-24T16:42:32,120 DEBUG [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:32,121 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.HStore(327): Store=10af3cc4e904fab18c276c01940de289/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:42:32,121 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,122 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:42:32,122 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10af3cc4e904fab18c276c01940de289 columnFamilyName B 2024-11-24T16:42:32,122 DEBUG [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:32,122 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.HStore(327): Store=10af3cc4e904fab18c276c01940de289/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:42:32,122 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,123 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:42:32,123 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10af3cc4e904fab18c276c01940de289 columnFamilyName C 2024-11-24T16:42:32,123 DEBUG [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:42:32,123 INFO [StoreOpener-10af3cc4e904fab18c276c01940de289-1 {}] regionserver.HStore(327): Store=10af3cc4e904fab18c276c01940de289/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:42:32,123 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:32,123 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,124 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,124 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,124 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,124 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,125 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:42:32,125 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,127 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:42:32,127 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 10af3cc4e904fab18c276c01940de289; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73907714, jitterRate=0.10131075978279114}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:42:32,127 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:32,128 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 10af3cc4e904fab18c276c01940de289: Running coprocessor pre-open hook at 1732466552118Writing region info on filesystem at 1732466552118Initializing all the Stores at 1732466552119 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466552119Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466552119Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466552119Cleaning up temporary data from old regions at 1732466552124 (+5 ms)Running coprocessor post-open hooks at 1732466552127 (+3 ms)Region opened successfully at 1732466552128 (+1 ms) 2024-11-24T16:42:32,128 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., pid=122, masterSystemTime=1732466552115 2024-11-24T16:42:32,130 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:32,130 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:32,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=10af3cc4e904fab18c276c01940de289, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:42:32,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:42:32,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-24T16:42:32,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 in 169 msec 2024-11-24T16:42:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-24T16:42:32,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-24T16:42:32,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, ASSIGN in 323 msec 2024-11-24T16:42:32,135 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:42:32,135 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466552135"}]},"ts":"1732466552135"} 2024-11-24T16:42:32,136 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T16:42:32,137 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:42:32,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-24T16:42:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-24T16:42:33,144 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:33,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dacd8f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,146 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b36aa33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,148 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a4273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,148 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,151 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:42:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,37979,1732466430336 2024-11-24T16:42:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6fa7c972 2024-11-24T16:42:33,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:42:33,152 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:42:33,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72240ede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,153 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,154 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa75da9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,155 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,155 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4037a356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,156 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c90706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@607ef71a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,159 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fcf41ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,160 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc24b95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,161 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3169e469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d227aa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,163 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abe1d6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,165 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec4c3df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,166 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,166 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,166 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658709e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,167 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18bee5e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,168 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,168 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,169 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a90054f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,170 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@361923b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,171 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c41ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,172 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67e5a9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@afcb3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,175 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdc7354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:42:33,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:42:33,176 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:42:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:42:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:42:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761b328f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:42:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:42:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:33,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:42:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63fa6440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:42:33,178 DEBUG [Thread-2138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,178 DEBUG [Thread-2123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,178 DEBUG [Thread-2134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,178 DEBUG [Thread-2125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,178 DEBUG [Thread-2127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,179 DEBUG [Thread-2119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,179 DEBUG [Thread-2121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,179 DEBUG [Thread-2132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,180 DEBUG [Thread-2134 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,180 DEBUG [Thread-2130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,181 DEBUG [Thread-2134 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:33,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,181 DEBUG [Thread-2130 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,181 DEBUG [Thread-2130 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-24T16:42:33,186 DEBUG [Thread-2136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:42:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T16:42:33,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,188 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:33,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:33,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:42:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:42:33,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:42:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2] 2024-11-24T16:42:33,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/024dd6a0e9c14eb3b13a7649b81dec22 is 50, key is test_row_0/A:col10/1732466553191/Put/seqid=0 2024-11-24T16:42:33,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742301_1477 (size=14341) 2024-11-24T16:42:33,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466613217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466613220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466613221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466613227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466613228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T16:42:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466613333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466613333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466613344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466613344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466613344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T16:42:33,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466613544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466613544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466613564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466613564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466613564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/024dd6a0e9c14eb3b13a7649b81dec22 2024-11-24T16:42:33,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/49175b970ab44e80876ef151d7c6e3bd is 50, key is test_row_0/B:col10/1732466553191/Put/seqid=0 2024-11-24T16:42:33,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742302_1478 (size=12001) 2024-11-24T16:42:33,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T16:42:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466613863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466613864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466613884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466613884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466613884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:33,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:33,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:33,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/49175b970ab44e80876ef151d7c6e3bd 2024-11-24T16:42:34,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/4df72c8204bf46248fe1f65c57c2e565 is 50, key is test_row_0/C:col10/1732466553191/Put/seqid=0 2024-11-24T16:42:34,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742303_1479 (size=12001) 2024-11-24T16:42:34,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:34,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:34,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:34,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:34,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T16:42:34,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466614384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:34,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466614384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:34,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466614393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:34,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466614393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:34,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466614394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:34,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:34,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:34,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:34,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:34,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/4df72c8204bf46248fe1f65c57c2e565 2024-11-24T16:42:34,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/024dd6a0e9c14eb3b13a7649b81dec22 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/024dd6a0e9c14eb3b13a7649b81dec22 2024-11-24T16:42:34,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/024dd6a0e9c14eb3b13a7649b81dec22, entries=200, sequenceid=13, filesize=14.0 K 2024-11-24T16:42:34,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/49175b970ab44e80876ef151d7c6e3bd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/49175b970ab44e80876ef151d7c6e3bd 2024-11-24T16:42:34,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/49175b970ab44e80876ef151d7c6e3bd, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T16:42:34,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/4df72c8204bf46248fe1f65c57c2e565 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/4df72c8204bf46248fe1f65c57c2e565 2024-11-24T16:42:34,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/4df72c8204bf46248fe1f65c57c2e565, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T16:42:34,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 10af3cc4e904fab18c276c01940de289 in 1306ms, sequenceid=13, compaction requested=false 2024-11-24T16:42:34,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:34,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T16:42:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:34,565 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:42:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:34,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/e7408797e24e46e188ae90411e8589ea is 50, key is test_row_0/A:col10/1732466553220/Put/seqid=0 2024-11-24T16:42:34,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742304_1480 (size=12001) 2024-11-24T16:42:34,577 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/e7408797e24e46e188ae90411e8589ea 2024-11-24T16:42:34,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/8fc85074ab884dd0954739c665952f7f is 50, key is test_row_0/B:col10/1732466553220/Put/seqid=0 2024-11-24T16:42:34,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742305_1481 (size=12001) 2024-11-24T16:42:34,596 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/8fc85074ab884dd0954739c665952f7f 2024-11-24T16:42:34,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/f24e7f7ff58c42569068e6e073afd1d0 is 50, key is test_row_0/C:col10/1732466553220/Put/seqid=0 2024-11-24T16:42:34,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742306_1482 (size=12001) 2024-11-24T16:42:35,015 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/f24e7f7ff58c42569068e6e073afd1d0 2024-11-24T16:42:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/e7408797e24e46e188ae90411e8589ea as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/e7408797e24e46e188ae90411e8589ea 2024-11-24T16:42:35,023 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/e7408797e24e46e188ae90411e8589ea, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T16:42:35,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/8fc85074ab884dd0954739c665952f7f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8fc85074ab884dd0954739c665952f7f 2024-11-24T16:42:35,026 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8fc85074ab884dd0954739c665952f7f, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T16:42:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/f24e7f7ff58c42569068e6e073afd1d0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f24e7f7ff58c42569068e6e073afd1d0 2024-11-24T16:42:35,032 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f24e7f7ff58c42569068e6e073afd1d0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T16:42:35,034 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 10af3cc4e904fab18c276c01940de289 in 469ms, sequenceid=33, compaction requested=false 2024-11-24T16:42:35,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:35,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-24T16:42:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-24T16:42:35,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-24T16:42:35,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8480 sec 2024-11-24T16:42:35,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.8570 sec 2024-11-24T16:42:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T16:42:35,334 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:35,334 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-24T16:42:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T16:42:35,336 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:35,337 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:35,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:35,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:35,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:35,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:35,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:35,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:35,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:35,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:35,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/f60b7d791a1b40438bf59f1ab5b3d3b4 is 50, key is test_row_0/A:col10/1732466555401/Put/seqid=0 2024-11-24T16:42:35,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742307_1483 (size=14341) 2024-11-24T16:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T16:42:35,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466555437, max timestamp=1732466555456Num uniques 859; 2024-11-24T16:42:35,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:35,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466555437, max timestamp=1732466555456Num uniques 859; 2024-11-24T16:42:35,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732466555437, max timestamp=1732466555456Num uniques 859; 2024-11-24T16:42:35,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:35,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T16:42:35,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466615480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466615485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466615491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466615493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466615494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466615604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466615604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466615604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466615613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466615613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T16:42:35,713 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:42:35,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/f60b7d791a1b40438bf59f1ab5b3d3b4 2024-11-24T16:42:35,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/8eb7d839c08a4be2953afd08e6708876 is 50, key is test_row_0/B:col10/1732466555401/Put/seqid=0 2024-11-24T16:42:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742308_1484 (size=12001) 2024-11-24T16:42:35,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466615823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466615823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466615823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466615834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466615834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:35,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:35,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:35,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T16:42:36,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:36,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:36,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466616144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466616144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466616144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466616154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466616154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/8eb7d839c08a4be2953afd08e6708876 2024-11-24T16:42:36,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/8c430ad06a904c67bd3152ce6af1b6f6 is 50, key is test_row_0/C:col10/1732466555401/Put/seqid=0 2024-11-24T16:42:36,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742309_1485 (size=12001) 2024-11-24T16:42:36,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:36,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:36,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:36,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T16:42:36,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:36,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/8c430ad06a904c67bd3152ce6af1b6f6 2024-11-24T16:42:36,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/f60b7d791a1b40438bf59f1ab5b3d3b4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/f60b7d791a1b40438bf59f1ab5b3d3b4 2024-11-24T16:42:36,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/f60b7d791a1b40438bf59f1ab5b3d3b4, entries=200, sequenceid=44, filesize=14.0 K 2024-11-24T16:42:36,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/8eb7d839c08a4be2953afd08e6708876 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8eb7d839c08a4be2953afd08e6708876 2024-11-24T16:42:36,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8eb7d839c08a4be2953afd08e6708876, entries=150, sequenceid=44, filesize=11.7 K 2024-11-24T16:42:36,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/8c430ad06a904c67bd3152ce6af1b6f6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8c430ad06a904c67bd3152ce6af1b6f6 2024-11-24T16:42:36,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8c430ad06a904c67bd3152ce6af1b6f6, entries=150, sequenceid=44, filesize=11.7 K 2024-11-24T16:42:36,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 10af3cc4e904fab18c276c01940de289 in 1249ms, sequenceid=44, compaction requested=true 2024-11-24T16:42:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:36,662 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:36,662 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:36,663 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:36,663 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/A is initiating minor compaction (all files) 2024-11-24T16:42:36,663 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/A in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,663 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/024dd6a0e9c14eb3b13a7649b81dec22, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/e7408797e24e46e188ae90411e8589ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/f60b7d791a1b40438bf59f1ab5b3d3b4] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=39.7 K 2024-11-24T16:42:36,663 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:36,663 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/B is initiating minor compaction (all files) 2024-11-24T16:42:36,663 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/B in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,663 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/49175b970ab44e80876ef151d7c6e3bd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8fc85074ab884dd0954739c665952f7f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8eb7d839c08a4be2953afd08e6708876] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=35.2 K 2024-11-24T16:42:36,664 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024dd6a0e9c14eb3b13a7649b81dec22, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466553186 2024-11-24T16:42:36,664 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 49175b970ab44e80876ef151d7c6e3bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466553186 2024-11-24T16:42:36,664 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7408797e24e46e188ae90411e8589ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732466553220 2024-11-24T16:42:36,664 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc85074ab884dd0954739c665952f7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732466553220 2024-11-24T16:42:36,664 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60b7d791a1b40438bf59f1ab5b3d3b4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732466555401 2024-11-24T16:42:36,665 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eb7d839c08a4be2953afd08e6708876, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732466555401 2024-11-24T16:42:36,673 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:36,673 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/c447e26923fb4ec29328fc0a99787842 is 50, key is test_row_0/B:col10/1732466555401/Put/seqid=0 2024-11-24T16:42:36,673 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:36,674 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/5770df3f1b0240c59763b96f54a97be7 is 50, key is test_row_0/A:col10/1732466555401/Put/seqid=0 2024-11-24T16:42:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:36,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T16:42:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:36,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:36,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742311_1487 (size=12104) 2024-11-24T16:42:36,686 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/5770df3f1b0240c59763b96f54a97be7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5770df3f1b0240c59763b96f54a97be7 2024-11-24T16:42:36,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742310_1486 (size=12104) 2024-11-24T16:42:36,690 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/A of 10af3cc4e904fab18c276c01940de289 into 5770df3f1b0240c59763b96f54a97be7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:36,690 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:36,690 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/A, priority=13, startTime=1732466556662; duration=0sec 2024-11-24T16:42:36,690 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:36,690 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:A 2024-11-24T16:42:36,690 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:36,691 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:36,691 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/C is initiating minor compaction (all files) 2024-11-24T16:42:36,691 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/C in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,691 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/4df72c8204bf46248fe1f65c57c2e565, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f24e7f7ff58c42569068e6e073afd1d0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8c430ad06a904c67bd3152ce6af1b6f6] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=35.2 K 2024-11-24T16:42:36,692 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4df72c8204bf46248fe1f65c57c2e565, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732466553186 2024-11-24T16:42:36,692 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting f24e7f7ff58c42569068e6e073afd1d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732466553220 2024-11-24T16:42:36,693 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c430ad06a904c67bd3152ce6af1b6f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732466555401 2024-11-24T16:42:36,695 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/c447e26923fb4ec29328fc0a99787842 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/c447e26923fb4ec29328fc0a99787842 2024-11-24T16:42:36,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3d4b4937823c4faaa3e3961e8373e99f is 50, key is test_row_0/A:col10/1732466555492/Put/seqid=0 2024-11-24T16:42:36,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742312_1488 (size=14341) 2024-11-24T16:42:36,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,704 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466616685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466616687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,705 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/ae3714240591418cb0e01de0e6a9f380 is 50, key is test_row_0/C:col10/1732466555401/Put/seqid=0 2024-11-24T16:42:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,706 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/B of 10af3cc4e904fab18c276c01940de289 into c447e26923fb4ec29328fc0a99787842(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:36,706 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:36,706 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/B, priority=13, startTime=1732466556662; duration=0sec 2024-11-24T16:42:36,706 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:36,706 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:B 2024-11-24T16:42:36,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742313_1489 (size=12104) 2024-11-24T16:42:36,713 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/ae3714240591418cb0e01de0e6a9f380 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/ae3714240591418cb0e01de0e6a9f380 2024-11-24T16:42:36,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:36,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:36,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466616697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466616699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466616699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,717 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/C of 10af3cc4e904fab18c276c01940de289 into ae3714240591418cb0e01de0e6a9f380(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:36,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,717 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:36,718 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/C, priority=13, startTime=1732466556662; duration=0sec 2024-11-24T16:42:36,718 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:36,718 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:C 2024-11-24T16:42:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466616814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466616814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466616824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466616824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466616824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:36,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:36,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:36,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:36,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:36,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:37,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:37,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:37,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:37,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466617024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466617024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466617044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466617044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466617045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3d4b4937823c4faaa3e3961e8373e99f 2024-11-24T16:42:37,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/bc38899d3aba490b844f023a2ebae7fd is 50, key is test_row_0/B:col10/1732466555492/Put/seqid=0 2024-11-24T16:42:37,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742314_1490 (size=12001) 2024-11-24T16:42:37,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/bc38899d3aba490b844f023a2ebae7fd 2024-11-24T16:42:37,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/a23b66fba6a948259f4fd2fdb8890401 is 50, key is test_row_0/C:col10/1732466555492/Put/seqid=0 2024-11-24T16:42:37,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742315_1491 (size=12001) 2024-11-24T16:42:37,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/a23b66fba6a948259f4fd2fdb8890401 2024-11-24T16:42:37,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3d4b4937823c4faaa3e3961e8373e99f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d4b4937823c4faaa3e3961e8373e99f 2024-11-24T16:42:37,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d4b4937823c4faaa3e3961e8373e99f, entries=200, sequenceid=79, filesize=14.0 K 2024-11-24T16:42:37,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/bc38899d3aba490b844f023a2ebae7fd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bc38899d3aba490b844f023a2ebae7fd 2024-11-24T16:42:37,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bc38899d3aba490b844f023a2ebae7fd, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T16:42:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/a23b66fba6a948259f4fd2fdb8890401 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/a23b66fba6a948259f4fd2fdb8890401 2024-11-24T16:42:37,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/a23b66fba6a948259f4fd2fdb8890401, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T16:42:37,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 10af3cc4e904fab18c276c01940de289 in 475ms, sequenceid=79, compaction requested=false 2024-11-24T16:42:37,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:37,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T16:42:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:37,174 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:37,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:37,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/b6439c796b8f4082baa9f9598b15cf64 is 50, key is test_row_0/A:col10/1732466556695/Put/seqid=0 2024-11-24T16:42:37,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742316_1492 (size=12001) 2024-11-24T16:42:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:37,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:37,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466557383, max timestamp=1732466557400Num uniques 859; 2024-11-24T16:42:37,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466557383, max timestamp=1732466557400Num uniques 859; 2024-11-24T16:42:37,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466557383, max timestamp=1732466557400Num uniques 859; 2024-11-24T16:42:37,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:37,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:37,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:37,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466617468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T16:42:37,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466617469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466617470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466617471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466617472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,585 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/b6439c796b8f4082baa9f9598b15cf64 2024-11-24T16:42:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466617583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1841245877af4e31ae016cc5e20e1dff is 50, key is test_row_0/B:col10/1732466556695/Put/seqid=0 2024-11-24T16:42:37,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742317_1493 (size=12001) 2024-11-24T16:42:37,596 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1841245877af4e31ae016cc5e20e1dff 2024-11-24T16:42:37,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/bc133f6879704edab5c0540bac73c8d0 is 50, key is test_row_0/C:col10/1732466556695/Put/seqid=0 2024-11-24T16:42:37,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742318_1494 (size=12001) 2024-11-24T16:42:37,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466617594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466617594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466617594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466617594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466617793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466617814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466617814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466617814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466617814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,006 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/bc133f6879704edab5c0540bac73c8d0 2024-11-24T16:42:38,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/b6439c796b8f4082baa9f9598b15cf64 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/b6439c796b8f4082baa9f9598b15cf64 2024-11-24T16:42:38,012 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/b6439c796b8f4082baa9f9598b15cf64, entries=150, sequenceid=92, filesize=11.7 K 2024-11-24T16:42:38,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1841245877af4e31ae016cc5e20e1dff as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1841245877af4e31ae016cc5e20e1dff 2024-11-24T16:42:38,016 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1841245877af4e31ae016cc5e20e1dff, entries=150, sequenceid=92, filesize=11.7 K 2024-11-24T16:42:38,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/bc133f6879704edab5c0540bac73c8d0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/bc133f6879704edab5c0540bac73c8d0 2024-11-24T16:42:38,019 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/bc133f6879704edab5c0540bac73c8d0, entries=150, sequenceid=92, filesize=11.7 K 2024-11-24T16:42:38,020 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 10af3cc4e904fab18c276c01940de289 in 846ms, sequenceid=92, compaction requested=true 2024-11-24T16:42:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:38,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-24T16:42:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-24T16:42:38,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-24T16:42:38,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6850 sec 2024-11-24T16:42:38,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6890 sec 2024-11-24T16:42:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:38,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T16:42:38,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:38,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:38,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:38,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:38,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:38,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:38,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/4942636598374e778fec843d8c85e933 is 50, key is test_row_0/A:col10/1732466557471/Put/seqid=0 2024-11-24T16:42:38,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742319_1495 (size=14391) 2024-11-24T16:42:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466618139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466618140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466618141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466618142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466618143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466618254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466618263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466618263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466618263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466618263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466618463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466618483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466618483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466618484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466618484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/4942636598374e778fec843d8c85e933 2024-11-24T16:42:38,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/aa552c76ba2744d3a2efd97ff0b5bb1f is 50, key is test_row_0/B:col10/1732466557471/Put/seqid=0 2024-11-24T16:42:38,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742320_1496 (size=12051) 2024-11-24T16:42:38,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466618774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466618803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466618803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466618803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466618804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:38,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:38,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/aa552c76ba2744d3a2efd97ff0b5bb1f 2024-11-24T16:42:38,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/25a1a467ef1240d4a022bf0604d8ed47 is 50, key is test_row_0/C:col10/1732466557471/Put/seqid=0 2024-11-24T16:42:38,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742321_1497 (size=12051) 2024-11-24T16:42:39,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466619294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:39,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:39,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466619313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466619313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466619313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:39,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466619314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:39,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:39,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:39,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/25a1a467ef1240d4a022bf0604d8ed47 2024-11-24T16:42:39,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/4942636598374e778fec843d8c85e933 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4942636598374e778fec843d8c85e933 2024-11-24T16:42:39,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4942636598374e778fec843d8c85e933, entries=200, sequenceid=129, filesize=14.1 K 2024-11-24T16:42:39,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/aa552c76ba2744d3a2efd97ff0b5bb1f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/aa552c76ba2744d3a2efd97ff0b5bb1f 2024-11-24T16:42:39,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/aa552c76ba2744d3a2efd97ff0b5bb1f, entries=150, sequenceid=129, filesize=11.8 K 2024-11-24T16:42:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/25a1a467ef1240d4a022bf0604d8ed47 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/25a1a467ef1240d4a022bf0604d8ed47 2024-11-24T16:42:39,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/25a1a467ef1240d4a022bf0604d8ed47, entries=150, sequenceid=129, filesize=11.8 K 2024-11-24T16:42:39,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 10af3cc4e904fab18c276c01940de289 in 1248ms, sequenceid=129, compaction requested=true 2024-11-24T16:42:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:39,370 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:39,370 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:39,372 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:39,372 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52837 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:39,372 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/B is initiating minor compaction (all files) 2024-11-24T16:42:39,372 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/A is initiating minor compaction (all files) 2024-11-24T16:42:39,372 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/B in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:39,372 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/A in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:39,372 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/c447e26923fb4ec29328fc0a99787842, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bc38899d3aba490b844f023a2ebae7fd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1841245877af4e31ae016cc5e20e1dff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/aa552c76ba2744d3a2efd97ff0b5bb1f] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=47.0 K 2024-11-24T16:42:39,372 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5770df3f1b0240c59763b96f54a97be7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d4b4937823c4faaa3e3961e8373e99f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/b6439c796b8f4082baa9f9598b15cf64, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4942636598374e778fec843d8c85e933] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=51.6 K 2024-11-24T16:42:39,373 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5770df3f1b0240c59763b96f54a97be7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732466555401 2024-11-24T16:42:39,373 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c447e26923fb4ec29328fc0a99787842, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732466555401 2024-11-24T16:42:39,373 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4b4937823c4faaa3e3961e8373e99f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732466555474 2024-11-24T16:42:39,373 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting bc38899d3aba490b844f023a2ebae7fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732466555474 2024-11-24T16:42:39,373 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6439c796b8f4082baa9f9598b15cf64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732466556683 2024-11-24T16:42:39,373 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1841245877af4e31ae016cc5e20e1dff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732466556683 2024-11-24T16:42:39,374 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4942636598374e778fec843d8c85e933, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732466557468 2024-11-24T16:42:39,374 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting aa552c76ba2744d3a2efd97ff0b5bb1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732466557470 2024-11-24T16:42:39,395 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:39,396 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3156af4050554c679bf918802cadcc36 is 50, key is test_row_0/A:col10/1732466557471/Put/seqid=0 2024-11-24T16:42:39,406 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:39,407 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/d4bc36ffd019442e803e862b542f475f is 50, key is test_row_0/B:col10/1732466557471/Put/seqid=0 2024-11-24T16:42:39,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742322_1498 (size=12291) 2024-11-24T16:42:39,414 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3156af4050554c679bf918802cadcc36 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3156af4050554c679bf918802cadcc36 2024-11-24T16:42:39,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742323_1499 (size=12291) 2024-11-24T16:42:39,420 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10af3cc4e904fab18c276c01940de289/A of 10af3cc4e904fab18c276c01940de289 into 3156af4050554c679bf918802cadcc36(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:39,420 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:39,420 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/A, priority=12, startTime=1732466559370; duration=0sec 2024-11-24T16:42:39,420 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:39,420 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:A 2024-11-24T16:42:39,420 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:39,423 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:39,423 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/C is initiating minor compaction (all files) 2024-11-24T16:42:39,423 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/C in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:39,423 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/d4bc36ffd019442e803e862b542f475f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/d4bc36ffd019442e803e862b542f475f 2024-11-24T16:42:39,423 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/ae3714240591418cb0e01de0e6a9f380, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/a23b66fba6a948259f4fd2fdb8890401, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/bc133f6879704edab5c0540bac73c8d0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/25a1a467ef1240d4a022bf0604d8ed47] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=47.0 K 2024-11-24T16:42:39,424 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3714240591418cb0e01de0e6a9f380, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1732466555401 2024-11-24T16:42:39,424 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a23b66fba6a948259f4fd2fdb8890401, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732466555474 2024-11-24T16:42:39,424 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc133f6879704edab5c0540bac73c8d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732466556683 2024-11-24T16:42:39,425 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a1a467ef1240d4a022bf0604d8ed47, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732466557470 2024-11-24T16:42:39,429 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10af3cc4e904fab18c276c01940de289/B of 10af3cc4e904fab18c276c01940de289 into d4bc36ffd019442e803e862b542f475f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:39,429 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:39,429 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/B, priority=12, startTime=1732466559370; duration=0sec 2024-11-24T16:42:39,429 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:39,429 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:B 2024-11-24T16:42:39,437 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#C#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:39,437 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/67184210b601403a89252beab5f212ed is 50, key is test_row_0/C:col10/1732466557471/Put/seqid=0 2024-11-24T16:42:39,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742324_1500 (size=12291) 2024-11-24T16:42:39,445 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/67184210b601403a89252beab5f212ed as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/67184210b601403a89252beab5f212ed 2024-11-24T16:42:39,450 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10af3cc4e904fab18c276c01940de289/C of 10af3cc4e904fab18c276c01940de289 into 67184210b601403a89252beab5f212ed(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:39,450 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:39,450 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/C, priority=12, startTime=1732466559370; duration=0sec 2024-11-24T16:42:39,450 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:39,450 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:C 2024-11-24T16:42:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T16:42:39,494 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:39,494 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-24T16:42:39,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T16:42:39,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:39,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:39,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T16:42:39,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T16:42:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:39,650 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T16:42:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:39,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:39,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/5b338068caad441ca6a15f0b10ffdf3d is 50, key is test_row_0/A:col10/1732466558139/Put/seqid=0 2024-11-24T16:42:39,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742325_1501 (size=12151) 2024-11-24T16:42:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T16:42:40,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/5b338068caad441ca6a15f0b10ffdf3d 2024-11-24T16:42:40,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/220763793f6c4a4ca7bd6d67b7f87c22 is 50, key is test_row_0/B:col10/1732466558139/Put/seqid=0 2024-11-24T16:42:40,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742326_1502 (size=12151) 2024-11-24T16:42:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T16:42:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:40,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732466560345, max timestamp=1732466560360Num uniques 859; 2024-11-24T16:42:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732466560345, max timestamp=1732466560360Num uniques 859; 2024-11-24T16:42:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T16:42:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T16:42:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732466560345, max timestamp=1732466560360Num uniques 859; 2024-11-24T16:42:40,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T16:42:40,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466620393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466620394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466620397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466620408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466620409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,475 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/220763793f6c4a4ca7bd6d67b7f87c22 2024-11-24T16:42:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/640c18e28649434d8d690ab3b8ec0f2a is 50, key is test_row_0/C:col10/1732466558139/Put/seqid=0 2024-11-24T16:42:40,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742327_1503 (size=12151) 2024-11-24T16:42:40,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466620513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466620513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466620513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466620524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466620534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T16:42:40,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466620734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466620734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466620734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466620743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466620743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:40,789 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T16:42:40,790 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T16:42:40,885 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/640c18e28649434d8d690ab3b8ec0f2a 2024-11-24T16:42:40,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/5b338068caad441ca6a15f0b10ffdf3d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5b338068caad441ca6a15f0b10ffdf3d 2024-11-24T16:42:40,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5b338068caad441ca6a15f0b10ffdf3d, entries=150, sequenceid=141, filesize=11.9 K 2024-11-24T16:42:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/220763793f6c4a4ca7bd6d67b7f87c22 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/220763793f6c4a4ca7bd6d67b7f87c22 2024-11-24T16:42:40,897 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/220763793f6c4a4ca7bd6d67b7f87c22, entries=150, sequenceid=141, filesize=11.9 K 2024-11-24T16:42:40,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/640c18e28649434d8d690ab3b8ec0f2a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/640c18e28649434d8d690ab3b8ec0f2a 2024-11-24T16:42:40,900 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/640c18e28649434d8d690ab3b8ec0f2a, entries=150, sequenceid=141, filesize=11.9 K 2024-11-24T16:42:40,901 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for 10af3cc4e904fab18c276c01940de289 in 1251ms, sequenceid=141, compaction requested=false 2024-11-24T16:42:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:40,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-24T16:42:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-24T16:42:40,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-24T16:42:40,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-24T16:42:40,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4110 sec 2024-11-24T16:42:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:41,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=161.02 KB heapSize=445.92 KB 2024-11-24T16:42:41,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:41,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:41,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:41,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:41,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:41,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:41,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/4a5799240ba0400ea480424a14d0a21c is 50, key is test_row_0/A:col10/1732466560407/Put/seqid=0 2024-11-24T16:42:41,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742328_1504 (size=14541) 2024-11-24T16:42:41,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466621063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466621063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466621064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466621068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466621077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466621183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466621183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466621194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466621404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466621404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466621404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/4a5799240ba0400ea480424a14d0a21c 2024-11-24T16:42:41,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/5149bc9eb47143398c274e7ac30584a0 is 50, key is test_row_0/B:col10/1732466560407/Put/seqid=0 2024-11-24T16:42:41,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742329_1505 (size=12151) 2024-11-24T16:42:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466621584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466621584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,594 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T16:42:41,644 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:41,644 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-24T16:42:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T16:42:41,646 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:41,647 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:41,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:41,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466621723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466621723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466621723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T16:42:41,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T16:42:41,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:41,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:41,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:41,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:41,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:41,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/5149bc9eb47143398c274e7ac30584a0 2024-11-24T16:42:41,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1ad912106aeb4f9fa80d7d26a952eea3 is 50, key is test_row_0/C:col10/1732466560407/Put/seqid=0 2024-11-24T16:42:41,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742330_1506 (size=12151) 2024-11-24T16:42:41,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T16:42:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:41,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:41,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T16:42:42,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T16:42:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:42,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:42,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466622243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466622243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466622244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:42,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:42,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T16:42:42,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:42,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:42,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:42,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T16:42:42,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1ad912106aeb4f9fa80d7d26a952eea3 2024-11-24T16:42:42,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/4a5799240ba0400ea480424a14d0a21c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4a5799240ba0400ea480424a14d0a21c 2024-11-24T16:42:42,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4a5799240ba0400ea480424a14d0a21c, entries=200, sequenceid=178, filesize=14.2 K 2024-11-24T16:42:42,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/5149bc9eb47143398c274e7ac30584a0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/5149bc9eb47143398c274e7ac30584a0 2024-11-24T16:42:42,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/5149bc9eb47143398c274e7ac30584a0, entries=150, sequenceid=178, filesize=11.9 K 2024-11-24T16:42:42,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1ad912106aeb4f9fa80d7d26a952eea3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1ad912106aeb4f9fa80d7d26a952eea3 2024-11-24T16:42:42,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1ad912106aeb4f9fa80d7d26a952eea3, entries=150, sequenceid=178, filesize=11.9 K 2024-11-24T16:42:42,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=26.84 KB/27480 for 10af3cc4e904fab18c276c01940de289 in 1247ms, sequenceid=178, compaction requested=true 2024-11-24T16:42:42,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:42,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:42,309 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:42,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T16:42:42,309 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:42,309 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:42,309 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38983 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:42,310 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/B is initiating minor compaction (all files) 2024-11-24T16:42:42,310 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/A is initiating minor compaction (all files) 2024-11-24T16:42:42,310 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/B in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,310 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/A in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,310 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/d4bc36ffd019442e803e862b542f475f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/220763793f6c4a4ca7bd6d67b7f87c22, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/5149bc9eb47143398c274e7ac30584a0] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=35.7 K 2024-11-24T16:42:42,310 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3156af4050554c679bf918802cadcc36, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5b338068caad441ca6a15f0b10ffdf3d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4a5799240ba0400ea480424a14d0a21c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=38.1 K 2024-11-24T16:42:42,310 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3156af4050554c679bf918802cadcc36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732466557470 2024-11-24T16:42:42,310 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bc36ffd019442e803e862b542f475f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732466557470 2024-11-24T16:42:42,310 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 220763793f6c4a4ca7bd6d67b7f87c22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732466558139 2024-11-24T16:42:42,310 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b338068caad441ca6a15f0b10ffdf3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732466558139 2024-11-24T16:42:42,311 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5149bc9eb47143398c274e7ac30584a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732466560407 2024-11-24T16:42:42,311 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5799240ba0400ea480424a14d0a21c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732466560378 2024-11-24T16:42:42,317 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:42,317 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/0310d80522ff4402bddb71ff905a333a is 50, key is test_row_0/B:col10/1732466560407/Put/seqid=0 2024-11-24T16:42:42,319 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:42,319 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/c77c72a7495b4d07a0a92146b699a114 is 50, key is test_row_0/A:col10/1732466560407/Put/seqid=0 2024-11-24T16:42:42,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742332_1508 (size=12493) 2024-11-24T16:42:42,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742331_1507 (size=12493) 2024-11-24T16:42:42,330 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/0310d80522ff4402bddb71ff905a333a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/0310d80522ff4402bddb71ff905a333a 2024-11-24T16:42:42,334 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/B of 10af3cc4e904fab18c276c01940de289 into 0310d80522ff4402bddb71ff905a333a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:42,334 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:42,334 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/B, priority=13, startTime=1732466562309; duration=0sec 2024-11-24T16:42:42,334 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:42,334 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:B 2024-11-24T16:42:42,334 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:42,340 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:42,341 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/C is initiating minor compaction (all files) 2024-11-24T16:42:42,341 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/C in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,341 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/67184210b601403a89252beab5f212ed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/640c18e28649434d8d690ab3b8ec0f2a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1ad912106aeb4f9fa80d7d26a952eea3] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=35.7 K 2024-11-24T16:42:42,341 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 67184210b601403a89252beab5f212ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732466557470 2024-11-24T16:42:42,342 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 640c18e28649434d8d690ab3b8ec0f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732466558139 2024-11-24T16:42:42,343 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ad912106aeb4f9fa80d7d26a952eea3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732466560407 2024-11-24T16:42:42,349 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:42,350 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/7fd696bad0644740b4d9fbb9428c5be0 is 50, key is test_row_0/C:col10/1732466560407/Put/seqid=0 2024-11-24T16:42:42,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742333_1509 (size=12493) 2024-11-24T16:42:42,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T16:42:42,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,411 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T16:42:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:42,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:42,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/19967ba3eb9d4b5ba3c9791ff96839bf is 50, key is test_row_0/A:col10/1732466561075/Put/seqid=0 2024-11-24T16:42:42,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742334_1510 (size=9757) 2024-11-24T16:42:42,419 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/19967ba3eb9d4b5ba3c9791ff96839bf 2024-11-24T16:42:42,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/3e7a89611e754bc1a516b466946405cf is 50, key is test_row_0/B:col10/1732466561075/Put/seqid=0 2024-11-24T16:42:42,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742335_1511 (size=9757) 2024-11-24T16:42:42,465 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/3e7a89611e754bc1a516b466946405cf 2024-11-24T16:42:42,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/f444a1a33eb04c63bb6c84b77fa7aa35 is 50, key is test_row_0/C:col10/1732466561075/Put/seqid=0 2024-11-24T16:42:42,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742336_1512 (size=9757) 2024-11-24T16:42:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:42,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466562628, max timestamp=1732466562667Num uniques 859; 2024-11-24T16:42:42,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466562628, max timestamp=1732466562667Num uniques 859; 2024-11-24T16:42:42,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:42,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:42,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466562628, max timestamp=1732466562667Num uniques 859; 2024-11-24T16:42:42,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:42,727 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/c77c72a7495b4d07a0a92146b699a114 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/c77c72a7495b4d07a0a92146b699a114 2024-11-24T16:42:42,731 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/A of 10af3cc4e904fab18c276c01940de289 into c77c72a7495b4d07a0a92146b699a114(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:42,731 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:42,731 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/A, priority=13, startTime=1732466562308; duration=0sec 2024-11-24T16:42:42,731 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:42,731 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:A 2024-11-24T16:42:42,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466562719, max timestamp=1732466562745Num uniques 1068; 2024-11-24T16:42:42,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466562719, max timestamp=1732466562745Num uniques 1068; 2024-11-24T16:42:42,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466562719, max timestamp=1732466562745Num uniques 1068; 2024-11-24T16:42:42,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-24T16:42:42,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-24T16:42:42,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-24T16:42:42,757 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/7fd696bad0644740b4d9fbb9428c5be0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7fd696bad0644740b4d9fbb9428c5be0 2024-11-24T16:42:42,760 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/C of 10af3cc4e904fab18c276c01940de289 into 7fd696bad0644740b4d9fbb9428c5be0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:42,760 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:42,760 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/C, priority=13, startTime=1732466562309; duration=0sec 2024-11-24T16:42:42,761 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:42,761 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:C 2024-11-24T16:42:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T16:42:42,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732466562760, max timestamp=1732466562819Num uniques 1027; 2024-11-24T16:42:42,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T16:42:42,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732466562760, max timestamp=1732466562819Num uniques 1027; 2024-11-24T16:42:42,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T16:42:42,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732466562760, max timestamp=1732466562819Num uniques 1027; 2024-11-24T16:42:42,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T16:42:42,886 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/f444a1a33eb04c63bb6c84b77fa7aa35 2024-11-24T16:42:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/19967ba3eb9d4b5ba3c9791ff96839bf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/19967ba3eb9d4b5ba3c9791ff96839bf 2024-11-24T16:42:42,895 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/19967ba3eb9d4b5ba3c9791ff96839bf, entries=100, sequenceid=186, filesize=9.5 K 2024-11-24T16:42:42,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/3e7a89611e754bc1a516b466946405cf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3e7a89611e754bc1a516b466946405cf 2024-11-24T16:42:42,898 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3e7a89611e754bc1a516b466946405cf, entries=100, sequenceid=186, filesize=9.5 K 2024-11-24T16:42:42,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/f444a1a33eb04c63bb6c84b77fa7aa35 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f444a1a33eb04c63bb6c84b77fa7aa35 2024-11-24T16:42:42,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732466562885, max timestamp=1732466562902Num uniques 1086; 2024-11-24T16:42:42,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-24T16:42:42,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732466562885, max timestamp=1732466562902Num uniques 1086; 2024-11-24T16:42:42,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-24T16:42:42,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732466562885, max timestamp=1732466562902Num uniques 1086; 2024-11-24T16:42:42,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-24T16:42:42,909 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f444a1a33eb04c63bb6c84b77fa7aa35, entries=100, sequenceid=186, filesize=9.5 K 2024-11-24T16:42:42,910 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=40.25 KB/41220 for 10af3cc4e904fab18c276c01940de289 in 499ms, sequenceid=186, compaction requested=false 2024-11-24T16:42:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-24T16:42:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-24T16:42:42,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-24T16:42:42,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2640 sec 2024-11-24T16:42:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:42,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=53.67 KB heapSize=138.70 KB 2024-11-24T16:42:42,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:42,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:42,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:42,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:42,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:42,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:42,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.2690 sec 2024-11-24T16:42:42,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/8546caa1093d4269b1d86a041458af3f is 50, key is test_row_0/A:col10/1732466562907/Put/seqid=0 2024-11-24T16:42:42,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742337_1513 (size=17081) 2024-11-24T16:42:43,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466562980, max timestamp=1732466562999Num uniques 859; 2024-11-24T16:42:43,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466562980, max timestamp=1732466562999Num uniques 859; 2024-11-24T16:42:43,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466562980, max timestamp=1732466562999Num uniques 859; 2024-11-24T16:42:43,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:43,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:43,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466623098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466623099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466623214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466623214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466623253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466623253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466623254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,266 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:43,266 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,267 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/8546caa1093d4269b1d86a041458af3f 2024-11-24T16:42:43,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1d664bbb10524040a87b5eabee3d5e15 is 50, key is test_row_0/B:col10/1732466562907/Put/seqid=0 2024-11-24T16:42:43,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742338_1514 (size=12301) 2024-11-24T16:42:43,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466623434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466623434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1d664bbb10524040a87b5eabee3d5e15 2024-11-24T16:42:43,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/42df7d1b5c83418998b613370ff86b69 is 50, key is test_row_0/C:col10/1732466562907/Put/seqid=0 2024-11-24T16:42:43,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742339_1515 (size=12301) 2024-11-24T16:42:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466623753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466623753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:43,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T16:42:43,794 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:43,794 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-24T16:42:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T16:42:43,796 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:43,797 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:43,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:43,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T16:42:43,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T16:42:43,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:43,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:43,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:43,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:43,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:44,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T16:42:44,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:44,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:44,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:44,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T16:42:44,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/42df7d1b5c83418998b613370ff86b69 2024-11-24T16:42:44,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/8546caa1093d4269b1d86a041458af3f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/8546caa1093d4269b1d86a041458af3f 2024-11-24T16:42:44,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/8546caa1093d4269b1d86a041458af3f, entries=250, sequenceid=263, filesize=16.7 K 2024-11-24T16:42:44,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1d664bbb10524040a87b5eabee3d5e15 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1d664bbb10524040a87b5eabee3d5e15 2024-11-24T16:42:44,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1d664bbb10524040a87b5eabee3d5e15, entries=150, sequenceid=263, filesize=12.0 K 2024-11-24T16:42:44,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/42df7d1b5c83418998b613370ff86b69 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/42df7d1b5c83418998b613370ff86b69 2024-11-24T16:42:44,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/42df7d1b5c83418998b613370ff86b69, entries=150, sequenceid=263, filesize=12.0 K 2024-11-24T16:42:44,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~138.66 KB/141984, currentSize=134.18 KB/137400 for 10af3cc4e904fab18c276c01940de289 in 1246ms, sequenceid=263, compaction requested=true 2024-11-24T16:42:44,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:44,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:44,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:44,161 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:44,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:44,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:44,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:44,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:44,161 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:44,162 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:44,162 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:44,163 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/A is initiating minor compaction (all files) 2024-11-24T16:42:44,163 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/A in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:44,163 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/c77c72a7495b4d07a0a92146b699a114, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/19967ba3eb9d4b5ba3c9791ff96839bf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/8546caa1093d4269b1d86a041458af3f] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=38.4 K 2024-11-24T16:42:44,163 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/B is initiating minor compaction (all files) 2024-11-24T16:42:44,163 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/B in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:44,163 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/0310d80522ff4402bddb71ff905a333a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3e7a89611e754bc1a516b466946405cf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1d664bbb10524040a87b5eabee3d5e15] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=33.7 K 2024-11-24T16:42:44,163 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting c77c72a7495b4d07a0a92146b699a114, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732466560407 2024-11-24T16:42:44,163 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0310d80522ff4402bddb71ff905a333a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732466560407 2024-11-24T16:42:44,163 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19967ba3eb9d4b5ba3c9791ff96839bf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732466561066 2024-11-24T16:42:44,164 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7a89611e754bc1a516b466946405cf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732466561066 2024-11-24T16:42:44,164 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8546caa1093d4269b1d86a041458af3f, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732466562885 2024-11-24T16:42:44,164 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d664bbb10524040a87b5eabee3d5e15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732466562907 2024-11-24T16:42:44,169 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:44,170 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/bf24b9e807564a35b88f3d81a1b70240 is 50, key is test_row_0/B:col10/1732466562907/Put/seqid=0 2024-11-24T16:42:44,172 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:44,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742340_1516 (size=12745) 2024-11-24T16:42:44,173 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/0ddfe0350573470987bda22ad1525b56 is 50, key is test_row_0/A:col10/1732466562907/Put/seqid=0 2024-11-24T16:42:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742341_1517 (size=12745) 2024-11-24T16:42:44,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T16:42:44,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:44,255 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T16:42:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:44,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/876e4a6382e749db8b86c10c704aa805 is 50, key is test_row_0/A:col10/1732466563097/Put/seqid=0 2024-11-24T16:42:44,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742342_1518 (size=12301) 2024-11-24T16:42:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:44,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:44,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466624304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:44,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466624309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T16:42:44,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466624424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466624424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:44,577 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/bf24b9e807564a35b88f3d81a1b70240 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bf24b9e807564a35b88f3d81a1b70240 2024-11-24T16:42:44,582 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/B of 10af3cc4e904fab18c276c01940de289 into bf24b9e807564a35b88f3d81a1b70240(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:44,582 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:44,582 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/B, priority=13, startTime=1732466564161; duration=0sec 2024-11-24T16:42:44,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:44,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:B 2024-11-24T16:42:44,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:44,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:44,583 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/C is initiating minor compaction (all files) 2024-11-24T16:42:44,583 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/C in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:44,584 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7fd696bad0644740b4d9fbb9428c5be0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f444a1a33eb04c63bb6c84b77fa7aa35, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/42df7d1b5c83418998b613370ff86b69] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=33.7 K 2024-11-24T16:42:44,584 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd696bad0644740b4d9fbb9428c5be0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732466560407 2024-11-24T16:42:44,584 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/0ddfe0350573470987bda22ad1525b56 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/0ddfe0350573470987bda22ad1525b56 2024-11-24T16:42:44,584 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f444a1a33eb04c63bb6c84b77fa7aa35, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732466561066 2024-11-24T16:42:44,584 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 42df7d1b5c83418998b613370ff86b69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732466562907 2024-11-24T16:42:44,591 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/A of 10af3cc4e904fab18c276c01940de289 into 0ddfe0350573470987bda22ad1525b56(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:44,591 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:44,591 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/A, priority=13, startTime=1732466564161; duration=0sec 2024-11-24T16:42:44,591 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:44,591 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:A 2024-11-24T16:42:44,594 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:44,594 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/d29391bea2a04dcc92b20a290b3fa829 is 50, key is test_row_0/C:col10/1732466562907/Put/seqid=0 2024-11-24T16:42:44,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742343_1519 (size=12745) 2024-11-24T16:42:44,613 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/d29391bea2a04dcc92b20a290b3fa829 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/d29391bea2a04dcc92b20a290b3fa829 2024-11-24T16:42:44,617 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/C of 10af3cc4e904fab18c276c01940de289 into d29391bea2a04dcc92b20a290b3fa829(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:44,617 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:44,617 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/C, priority=13, startTime=1732466564161; duration=0sec 2024-11-24T16:42:44,617 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:44,617 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:C 2024-11-24T16:42:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466624634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466624634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:44,665 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/876e4a6382e749db8b86c10c704aa805 2024-11-24T16:42:44,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/44f93c6fc58641c9be5bcb4e5afdc391 is 50, key is test_row_0/B:col10/1732466563097/Put/seqid=0 2024-11-24T16:42:44,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742344_1520 (size=12301) 2024-11-24T16:42:44,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T16:42:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466624953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466624953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:44,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,079 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/44f93c6fc58641c9be5bcb4e5afdc391 2024-11-24T16:42:45,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/9ec81744ce194c85b3e991756688bf03 is 50, key is test_row_0/C:col10/1732466563097/Put/seqid=0 2024-11-24T16:42:45,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742345_1521 (size=12301) 2024-11-24T16:42:45,091 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/9ec81744ce194c85b3e991756688bf03 2024-11-24T16:42:45,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/876e4a6382e749db8b86c10c704aa805 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/876e4a6382e749db8b86c10c704aa805 2024-11-24T16:42:45,097 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/876e4a6382e749db8b86c10c704aa805, entries=150, sequenceid=299, filesize=12.0 K 2024-11-24T16:42:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/44f93c6fc58641c9be5bcb4e5afdc391 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/44f93c6fc58641c9be5bcb4e5afdc391 2024-11-24T16:42:45,100 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/44f93c6fc58641c9be5bcb4e5afdc391, entries=150, sequenceid=299, filesize=12.0 K 2024-11-24T16:42:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/9ec81744ce194c85b3e991756688bf03 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/9ec81744ce194c85b3e991756688bf03 2024-11-24T16:42:45,104 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/9ec81744ce194c85b3e991756688bf03, entries=150, sequenceid=299, filesize=12.0 K 2024-11-24T16:42:45,105 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 10af3cc4e904fab18c276c01940de289 in 850ms, sequenceid=299, compaction requested=false 2024-11-24T16:42:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-24T16:42:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-24T16:42:45,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-24T16:42:45,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3090 sec 2024-11-24T16:42:45,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3130 sec 2024-11-24T16:42:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:45,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:42:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:45,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:45,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/72b46652703b40d4ae083c9d4f0bd0df is 50, key is test_row_0/A:col10/1732466565288/Put/seqid=0 2024-11-24T16:42:45,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742346_1522 (size=14741) 2024-11-24T16:42:45,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466625405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466625407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466625408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466625473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466625473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466625514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466625524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466625524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/72b46652703b40d4ae083c9d4f0bd0df 2024-11-24T16:42:45,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/7538282e0a9b453cb58710949cab0b79 is 50, key is test_row_0/B:col10/1732466565288/Put/seqid=0 2024-11-24T16:42:45,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742347_1523 (size=12301) 2024-11-24T16:42:45,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466625723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466625744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466625744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T16:42:45,954 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:45,954 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-24T16:42:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T16:42:45,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:45,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:45,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466626043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T16:42:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466626064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466626064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/7538282e0a9b453cb58710949cab0b79 2024-11-24T16:42:46,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1b5dc2f9f32345b18988f9a9851bbeda is 50, key is test_row_0/C:col10/1732466565288/Put/seqid=0 2024-11-24T16:42:46,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742348_1524 (size=12301) 2024-11-24T16:42:46,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:46,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:46,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T16:42:46,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:46,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:46,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466626483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,489 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466626493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,504 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1b5dc2f9f32345b18988f9a9851bbeda 2024-11-24T16:42:46,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/72b46652703b40d4ae083c9d4f0bd0df as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/72b46652703b40d4ae083c9d4f0bd0df 2024-11-24T16:42:46,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/72b46652703b40d4ae083c9d4f0bd0df, entries=200, sequenceid=314, filesize=14.4 K 2024-11-24T16:42:46,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/7538282e0a9b453cb58710949cab0b79 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/7538282e0a9b453cb58710949cab0b79 2024-11-24T16:42:46,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/7538282e0a9b453cb58710949cab0b79, entries=150, sequenceid=314, filesize=12.0 K 2024-11-24T16:42:46,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1b5dc2f9f32345b18988f9a9851bbeda as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1b5dc2f9f32345b18988f9a9851bbeda 2024-11-24T16:42:46,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1b5dc2f9f32345b18988f9a9851bbeda, entries=150, sequenceid=314, filesize=12.0 K 2024-11-24T16:42:46,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 10af3cc4e904fab18c276c01940de289 in 1248ms, sequenceid=314, compaction requested=true 2024-11-24T16:42:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:46,536 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:46,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:46,536 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:46,537 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:46,537 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:46,537 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/B is initiating minor compaction (all files) 2024-11-24T16:42:46,537 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/A is initiating minor compaction (all files) 2024-11-24T16:42:46,537 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/B in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,537 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/A in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,537 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bf24b9e807564a35b88f3d81a1b70240, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/44f93c6fc58641c9be5bcb4e5afdc391, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/7538282e0a9b453cb58710949cab0b79] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=36.5 K 2024-11-24T16:42:46,537 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/0ddfe0350573470987bda22ad1525b56, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/876e4a6382e749db8b86c10c704aa805, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/72b46652703b40d4ae083c9d4f0bd0df] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=38.9 K 2024-11-24T16:42:46,538 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting bf24b9e807564a35b88f3d81a1b70240, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732466562907 2024-11-24T16:42:46,538 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ddfe0350573470987bda22ad1525b56, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732466562907 2024-11-24T16:42:46,538 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 876e4a6382e749db8b86c10c704aa805, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732466563048 2024-11-24T16:42:46,538 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 44f93c6fc58641c9be5bcb4e5afdc391, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732466563048 2024-11-24T16:42:46,538 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72b46652703b40d4ae083c9d4f0bd0df, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732466564303 2024-11-24T16:42:46,538 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 7538282e0a9b453cb58710949cab0b79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732466564303 2024-11-24T16:42:46,546 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#A#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:46,546 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/179f72114c414a36bad6aae404f57e25 is 50, key is test_row_0/A:col10/1732466565288/Put/seqid=0 2024-11-24T16:42:46,549 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#B#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:46,550 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/bd825d54895840c5997ea8a3639e4625 is 50, key is test_row_0/B:col10/1732466565288/Put/seqid=0 2024-11-24T16:42:46,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742350_1526 (size=12847) 2024-11-24T16:42:46,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742349_1525 (size=12847) 2024-11-24T16:42:46,560 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/179f72114c414a36bad6aae404f57e25 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/179f72114c414a36bad6aae404f57e25 2024-11-24T16:42:46,564 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/A of 10af3cc4e904fab18c276c01940de289 into 179f72114c414a36bad6aae404f57e25(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:46,564 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:46,564 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/A, priority=13, startTime=1732466566536; duration=0sec 2024-11-24T16:42:46,564 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:46,564 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:A 2024-11-24T16:42:46,564 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:46,566 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:46,566 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/C is initiating minor compaction (all files) 2024-11-24T16:42:46,566 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/C in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,567 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/d29391bea2a04dcc92b20a290b3fa829, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/9ec81744ce194c85b3e991756688bf03, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1b5dc2f9f32345b18988f9a9851bbeda] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=36.5 K 2024-11-24T16:42:46,567 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting d29391bea2a04dcc92b20a290b3fa829, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732466562907 2024-11-24T16:42:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:46,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T16:42:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:46,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:46,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:46,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:46,570 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec81744ce194c85b3e991756688bf03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732466563048 2024-11-24T16:42:46,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:46,571 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b5dc2f9f32345b18988f9a9851bbeda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732466564303 2024-11-24T16:42:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/88be0e4649ff461c863a58cd2c2b5826 is 50, key is test_row_0/A:col10/1732466565373/Put/seqid=0 2024-11-24T16:42:46,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742351_1527 (size=14741) 2024-11-24T16:42:46,579 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:46,580 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/38b9782533e34151bcdfc844dca5e27f is 50, key is test_row_0/C:col10/1732466565288/Put/seqid=0 2024-11-24T16:42:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742352_1528 (size=12847) 2024-11-24T16:42:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T16:42:46,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466626598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466626599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466626607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:46,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466626714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466626714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466626724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:46,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466626934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466626934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:46,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466626934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:46,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:46,962 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/bd825d54895840c5997ea8a3639e4625 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bd825d54895840c5997ea8a3639e4625 2024-11-24T16:42:46,966 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/B of 10af3cc4e904fab18c276c01940de289 into bd825d54895840c5997ea8a3639e4625(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:46,966 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:46,966 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/B, priority=13, startTime=1732466566536; duration=0sec 2024-11-24T16:42:46,966 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:46,966 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:B 2024-11-24T16:42:46,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/88be0e4649ff461c863a58cd2c2b5826 2024-11-24T16:42:46,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1a03ca0fd83643d39f1c1658bf0ff79b is 50, key is test_row_0/B:col10/1732466565373/Put/seqid=0 2024-11-24T16:42:46,987 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/38b9782533e34151bcdfc844dca5e27f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/38b9782533e34151bcdfc844dca5e27f 2024-11-24T16:42:46,991 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/C of 10af3cc4e904fab18c276c01940de289 into 38b9782533e34151bcdfc844dca5e27f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:46,991 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:46,991 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/C, priority=13, startTime=1732466566536; duration=0sec 2024-11-24T16:42:46,991 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:46,991 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:C 2024-11-24T16:42:46,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742353_1529 (size=12301) 2024-11-24T16:42:46,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1a03ca0fd83643d39f1c1658bf0ff79b 2024-11-24T16:42:47,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/5d6de0e8a402433c9309f75ac3e6892d is 50, key is test_row_0/C:col10/1732466565373/Put/seqid=0 2024-11-24T16:42:47,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742354_1530 (size=12301) 2024-11-24T16:42:47,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/5d6de0e8a402433c9309f75ac3e6892d 2024-11-24T16:42:47,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/88be0e4649ff461c863a58cd2c2b5826 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/88be0e4649ff461c863a58cd2c2b5826 2024-11-24T16:42:47,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/88be0e4649ff461c863a58cd2c2b5826, entries=200, sequenceid=336, filesize=14.4 K 2024-11-24T16:42:47,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/1a03ca0fd83643d39f1c1658bf0ff79b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1a03ca0fd83643d39f1c1658bf0ff79b 2024-11-24T16:42:47,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:47,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:47,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:47,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:47,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:47,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1a03ca0fd83643d39f1c1658bf0ff79b, entries=150, sequenceid=336, filesize=12.0 K 2024-11-24T16:42:47,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/5d6de0e8a402433c9309f75ac3e6892d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/5d6de0e8a402433c9309f75ac3e6892d 2024-11-24T16:42:47,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/5d6de0e8a402433c9309f75ac3e6892d, entries=150, sequenceid=336, filesize=12.0 K 2024-11-24T16:42:47,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 10af3cc4e904fab18c276c01940de289 in 479ms, sequenceid=336, compaction requested=false 2024-11-24T16:42:47,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T16:42:47,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T16:42:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:47,185 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:42:47,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:47,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:47,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:47,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:47,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:47,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:47,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/a1ffefb52b944dc493e7e4058236343c is 50, key is test_row_0/A:col10/1732466566597/Put/seqid=0 2024-11-24T16:42:47,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742355_1531 (size=12301) 2024-11-24T16:42:47,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:47,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466627306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466567295, max timestamp=1732466567319Num uniques 859; 2024-11-24T16:42:47,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466567295, max timestamp=1732466567319Num uniques 859; 2024-11-24T16:42:47,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:47,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:47,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466567295, max timestamp=1732466567319Num uniques 859; 2024-11-24T16:42:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466627313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466627434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466627440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466627447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466627553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466627564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,598 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/a1ffefb52b944dc493e7e4058236343c 2024-11-24T16:42:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/81f004877d134ee39f6a1dfa204c60df is 50, key is test_row_0/B:col10/1732466566597/Put/seqid=0 2024-11-24T16:42:47,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742356_1532 (size=12301) 2024-11-24T16:42:47,610 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/81f004877d134ee39f6a1dfa204c60df 2024-11-24T16:42:47,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/6bc10ab6e1494de58eb3d9d00c7d9184 is 50, key is test_row_0/C:col10/1732466566597/Put/seqid=0 2024-11-24T16:42:47,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742357_1533 (size=12301) 2024-11-24T16:42:47,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466627653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466627774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466627784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:47,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466627973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:47,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,019 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/6bc10ab6e1494de58eb3d9d00c7d9184 2024-11-24T16:42:48,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/a1ffefb52b944dc493e7e4058236343c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a1ffefb52b944dc493e7e4058236343c 2024-11-24T16:42:48,026 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a1ffefb52b944dc493e7e4058236343c, entries=150, sequenceid=349, filesize=12.0 K 2024-11-24T16:42:48,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/81f004877d134ee39f6a1dfa204c60df as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/81f004877d134ee39f6a1dfa204c60df 2024-11-24T16:42:48,029 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/81f004877d134ee39f6a1dfa204c60df, entries=150, sequenceid=349, filesize=12.0 K 2024-11-24T16:42:48,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/6bc10ab6e1494de58eb3d9d00c7d9184 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/6bc10ab6e1494de58eb3d9d00c7d9184 2024-11-24T16:42:48,033 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/6bc10ab6e1494de58eb3d9d00c7d9184, entries=150, sequenceid=349, filesize=12.0 K 2024-11-24T16:42:48,034 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 10af3cc4e904fab18c276c01940de289 in 850ms, sequenceid=349, compaction requested=true 2024-11-24T16:42:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-24T16:42:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-24T16:42:48,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-24T16:42:48,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0790 sec 2024-11-24T16:42:48,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0840 sec 2024-11-24T16:42:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:48,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T16:42:48,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:48,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:48,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:48,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:48,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:48,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:48,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/1f228f3211fc47b7987c575e0fc4273c is 50, key is test_row_0/A:col10/1732466568092/Put/seqid=0 2024-11-24T16:42:48,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742358_1534 (size=14741) 2024-11-24T16:42:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T16:42:48,104 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:48,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-24T16:42:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T16:42:48,106 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:48,107 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:48,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:48,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466628128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466628136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T16:42:48,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:48,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466628254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466628254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T16:42:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466628464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466628464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/1f228f3211fc47b7987c575e0fc4273c 2024-11-24T16:42:48,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466628493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/3d3d177d5f1f4d6db547df53cd7cd1a9 is 50, key is test_row_0/B:col10/1732466568092/Put/seqid=0 2024-11-24T16:42:48,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742359_1535 (size=12301) 2024-11-24T16:42:48,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466628514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,524 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466628524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,535 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:48,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T16:42:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466628784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466628784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:48,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:48,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:48,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:48,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/3d3d177d5f1f4d6db547df53cd7cd1a9 2024-11-24T16:42:48,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1cdb2d532e0d4d39b5207b8770baace6 is 50, key is test_row_0/C:col10/1732466568092/Put/seqid=0 2024-11-24T16:42:48,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742360_1536 (size=12301) 2024-11-24T16:42:49,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:49,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:49,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:49,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:49,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T16:42:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466629304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466629304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:49,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1cdb2d532e0d4d39b5207b8770baace6 2024-11-24T16:42:49,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/1f228f3211fc47b7987c575e0fc4273c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/1f228f3211fc47b7987c575e0fc4273c 2024-11-24T16:42:49,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:49,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/1f228f3211fc47b7987c575e0fc4273c, entries=200, sequenceid=385, filesize=14.4 K 2024-11-24T16:42:49,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/3d3d177d5f1f4d6db547df53cd7cd1a9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3d3d177d5f1f4d6db547df53cd7cd1a9 2024-11-24T16:42:49,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3d3d177d5f1f4d6db547df53cd7cd1a9, entries=150, sequenceid=385, filesize=12.0 K 2024-11-24T16:42:49,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/1cdb2d532e0d4d39b5207b8770baace6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1cdb2d532e0d4d39b5207b8770baace6 2024-11-24T16:42:49,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1cdb2d532e0d4d39b5207b8770baace6, entries=150, sequenceid=385, filesize=12.0 K 2024-11-24T16:42:49,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 10af3cc4e904fab18c276c01940de289 in 1247ms, sequenceid=385, compaction requested=true 2024-11-24T16:42:49,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:49,341 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:49,341 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:49,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:49,342 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54630 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:49,342 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/A is initiating minor compaction (all files) 2024-11-24T16:42:49,342 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/A in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,342 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/179f72114c414a36bad6aae404f57e25, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/88be0e4649ff461c863a58cd2c2b5826, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a1ffefb52b944dc493e7e4058236343c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/1f228f3211fc47b7987c575e0fc4273c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=53.3 K 2024-11-24T16:42:49,343 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:49,343 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 179f72114c414a36bad6aae404f57e25, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732466564303 2024-11-24T16:42:49,343 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/B is initiating minor compaction (all files) 2024-11-24T16:42:49,343 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/B in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,343 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bd825d54895840c5997ea8a3639e4625, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1a03ca0fd83643d39f1c1658bf0ff79b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/81f004877d134ee39f6a1dfa204c60df, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3d3d177d5f1f4d6db547df53cd7cd1a9] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=48.6 K 2024-11-24T16:42:49,343 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88be0e4649ff461c863a58cd2c2b5826, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732466565373 2024-11-24T16:42:49,343 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting bd825d54895840c5997ea8a3639e4625, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732466564303 2024-11-24T16:42:49,344 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1ffefb52b944dc493e7e4058236343c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732466566591 2024-11-24T16:42:49,344 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a03ca0fd83643d39f1c1658bf0ff79b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732466565373 2024-11-24T16:42:49,344 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f228f3211fc47b7987c575e0fc4273c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732466567377 2024-11-24T16:42:49,344 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f004877d134ee39f6a1dfa204c60df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732466566591 2024-11-24T16:42:49,344 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3d177d5f1f4d6db547df53cd7cd1a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732466567377 2024-11-24T16:42:49,353 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#A#compaction#458 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:49,353 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:49,353 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/ba5b3255c030488ca91ff9393b905c44 is 50, key is test_row_0/B:col10/1732466568092/Put/seqid=0 2024-11-24T16:42:49,353 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/524959d0329248f3b4c0341b0ae25553 is 50, key is test_row_0/A:col10/1732466568092/Put/seqid=0 2024-11-24T16:42:49,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742361_1537 (size=12983) 2024-11-24T16:42:49,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742362_1538 (size=12983) 2024-11-24T16:42:49,369 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/ba5b3255c030488ca91ff9393b905c44 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/ba5b3255c030488ca91ff9393b905c44 2024-11-24T16:42:49,371 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/524959d0329248f3b4c0341b0ae25553 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/524959d0329248f3b4c0341b0ae25553 2024-11-24T16:42:49,374 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10af3cc4e904fab18c276c01940de289/B of 10af3cc4e904fab18c276c01940de289 into ba5b3255c030488ca91ff9393b905c44(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:49,374 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:49,374 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/B, priority=12, startTime=1732466569341; duration=0sec 2024-11-24T16:42:49,374 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:49,374 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:B 2024-11-24T16:42:49,374 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:42:49,376 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:42:49,376 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/C is initiating minor compaction (all files) 2024-11-24T16:42:49,376 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/C in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,376 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/38b9782533e34151bcdfc844dca5e27f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/5d6de0e8a402433c9309f75ac3e6892d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/6bc10ab6e1494de58eb3d9d00c7d9184, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1cdb2d532e0d4d39b5207b8770baace6] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=48.6 K 2024-11-24T16:42:49,376 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b9782533e34151bcdfc844dca5e27f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732466564303 2024-11-24T16:42:49,377 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d6de0e8a402433c9309f75ac3e6892d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732466565373 2024-11-24T16:42:49,377 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc10ab6e1494de58eb3d9d00c7d9184, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732466566591 2024-11-24T16:42:49,377 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdb2d532e0d4d39b5207b8770baace6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732466567377 2024-11-24T16:42:49,378 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10af3cc4e904fab18c276c01940de289/A of 10af3cc4e904fab18c276c01940de289 into 524959d0329248f3b4c0341b0ae25553(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:49,378 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:49,379 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/A, priority=12, startTime=1732466569340; duration=0sec 2024-11-24T16:42:49,379 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:49,379 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:A 2024-11-24T16:42:49,386 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#C#compaction#460 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:49,386 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/104e4ed47a8741cebd9a995dea63f782 is 50, key is test_row_0/C:col10/1732466568092/Put/seqid=0 2024-11-24T16:42:49,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742363_1539 (size=12983) 2024-11-24T16:42:49,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T16:42:49,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:49,483 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:49,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/81d1552087434b489cdc904452e3b79f is 50, key is test_row_0/A:col10/1732466568134/Put/seqid=0 2024-11-24T16:42:49,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742364_1540 (size=9857) 2024-11-24T16:42:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:49,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466569636, max timestamp=1732466569679Num uniques 859; 2024-11-24T16:42:49,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:42:49,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466569636, max timestamp=1732466569679Num uniques 859; 2024-11-24T16:42:49,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:42:49,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466569636, max timestamp=1732466569679Num uniques 859; 2024-11-24T16:42:49,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:42:49,795 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/104e4ed47a8741cebd9a995dea63f782 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/104e4ed47a8741cebd9a995dea63f782 2024-11-24T16:42:49,800 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 10af3cc4e904fab18c276c01940de289/C of 10af3cc4e904fab18c276c01940de289 into 104e4ed47a8741cebd9a995dea63f782(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:49,800 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:49,800 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/C, priority=12, startTime=1732466569341; duration=0sec 2024-11-24T16:42:49,800 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:49,800 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:C 2024-11-24T16:42:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466629815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:49,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/81d1552087434b489cdc904452e3b79f 2024-11-24T16:42:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/f283b2c29cfa42b19358ce1d9081b55d is 50, key is test_row_0/B:col10/1732466568134/Put/seqid=0 2024-11-24T16:42:49,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742365_1541 (size=9857) 2024-11-24T16:42:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466629933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466630144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T16:42:50,304 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/f283b2c29cfa42b19358ce1d9081b55d 2024-11-24T16:42:50,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/7ee7511abe4d4ce3819e176cc328b363 is 50, key is test_row_0/C:col10/1732466568134/Put/seqid=0 2024-11-24T16:42:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742366_1542 (size=9857) 2024-11-24T16:42:50,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:50,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466630323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:50,335 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_2' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:50,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466630334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:50,344 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466630464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:50,714 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/7ee7511abe4d4ce3819e176cc328b363 2024-11-24T16:42:50,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/81d1552087434b489cdc904452e3b79f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/81d1552087434b489cdc904452e3b79f 2024-11-24T16:42:50,721 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/81d1552087434b489cdc904452e3b79f, entries=100, sequenceid=397, filesize=9.6 K 2024-11-24T16:42:50,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/f283b2c29cfa42b19358ce1d9081b55d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f283b2c29cfa42b19358ce1d9081b55d 2024-11-24T16:42:50,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f283b2c29cfa42b19358ce1d9081b55d, entries=100, sequenceid=397, filesize=9.6 K 2024-11-24T16:42:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/7ee7511abe4d4ce3819e176cc328b363 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7ee7511abe4d4ce3819e176cc328b363 2024-11-24T16:42:50,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7ee7511abe4d4ce3819e176cc328b363, entries=100, sequenceid=397, filesize=9.6 K 2024-11-24T16:42:50,730 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 10af3cc4e904fab18c276c01940de289 in 1247ms, sequenceid=397, compaction requested=false 2024-11-24T16:42:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-24T16:42:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-24T16:42:50,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-24T16:42:50,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-11-24T16:42:50,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.6300 sec 2024-11-24T16:42:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:50,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-24T16:42:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:50,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:42:50,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3d719b4b942a4db59bca4acd8f7b82bb is 50, key is test_row_0/A:col10/1732466569807/Put/seqid=0 2024-11-24T16:42:51,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742367_1543 (size=14741) 2024-11-24T16:42:51,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466631038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:51,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466631154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:51,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466631374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:51,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:51,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3d719b4b942a4db59bca4acd8f7b82bb 2024-11-24T16:42:51,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/660da995f48440d7818ca6834de18efc is 50, key is test_row_0/B:col10/1732466569807/Put/seqid=0 2024-11-24T16:42:51,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742368_1544 (size=12301) 2024-11-24T16:42:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466631694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:51,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/660da995f48440d7818ca6834de18efc 2024-11-24T16:42:51,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/8eedf671081f49db870af10820fa03eb is 50, key is test_row_0/C:col10/1732466569807/Put/seqid=0 2024-11-24T16:42:51,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742369_1545 (size=12301) 2024-11-24T16:42:52,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466632214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/8eedf671081f49db870af10820fa03eb 2024-11-24T16:42:52,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3d719b4b942a4db59bca4acd8f7b82bb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d719b4b942a4db59bca4acd8f7b82bb 2024-11-24T16:42:52,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d719b4b942a4db59bca4acd8f7b82bb, entries=200, sequenceid=436, filesize=14.4 K 2024-11-24T16:42:52,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/660da995f48440d7818ca6834de18efc as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/660da995f48440d7818ca6834de18efc 2024-11-24T16:42:52,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/660da995f48440d7818ca6834de18efc, entries=150, sequenceid=436, filesize=12.0 K 2024-11-24T16:42:52,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/8eedf671081f49db870af10820fa03eb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8eedf671081f49db870af10820fa03eb 2024-11-24T16:42:52,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8eedf671081f49db870af10820fa03eb, entries=150, sequenceid=436, filesize=12.0 K 2024-11-24T16:42:52,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 10af3cc4e904fab18c276c01940de289 in 1253ms, sequenceid=436, compaction requested=true 2024-11-24T16:42:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:52,247 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:42:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:52,247 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:42:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 10af3cc4e904fab18c276c01940de289:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:42:52,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:52,249 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:52,249 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/B is initiating minor compaction (all files) 2024-11-24T16:42:52,249 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/B in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,249 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/ba5b3255c030488ca91ff9393b905c44, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f283b2c29cfa42b19358ce1d9081b55d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/660da995f48440d7818ca6834de18efc] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=34.3 K 2024-11-24T16:42:52,249 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37581 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:52,249 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/A is initiating minor compaction (all files) 2024-11-24T16:42:52,249 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/A in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,249 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/524959d0329248f3b4c0341b0ae25553, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/81d1552087434b489cdc904452e3b79f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d719b4b942a4db59bca4acd8f7b82bb] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=36.7 K 2024-11-24T16:42:52,249 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ba5b3255c030488ca91ff9393b905c44, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732466567377 2024-11-24T16:42:52,250 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524959d0329248f3b4c0341b0ae25553, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732466567377 2024-11-24T16:42:52,250 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f283b2c29cfa42b19358ce1d9081b55d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466568126 2024-11-24T16:42:52,250 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81d1552087434b489cdc904452e3b79f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466568126 2024-11-24T16:42:52,250 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d719b4b942a4db59bca4acd8f7b82bb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732466569766 2024-11-24T16:42:52,250 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 660da995f48440d7818ca6834de18efc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732466569766 2024-11-24T16:42:52,262 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:52,263 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#A#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:52,263 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/f9efbf0f1bab4523b982250660669a86 is 50, key is test_row_0/B:col10/1732466569807/Put/seqid=0 2024-11-24T16:42:52,263 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/17ee3444e5564a208e070c2b94d103dd is 50, key is test_row_0/A:col10/1732466569807/Put/seqid=0 2024-11-24T16:42:52,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742370_1546 (size=13085) 2024-11-24T16:42:52,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742371_1547 (size=13085) 2024-11-24T16:42:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T16:42:52,284 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:52,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:42:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-24T16:42:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T16:42:52,286 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:42:52,287 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:42:52,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:42:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:52,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:42:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:52,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:42:52,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/749bedcf087a43d8b33432f70ecad3cc is 50, key is test_row_0/A:col10/1732466571037/Put/seqid=0 2024-11-24T16:42:52,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742372_1548 (size=14741) 2024-11-24T16:42:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T16:42:52,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466572381, max timestamp=1732466572431Num uniques 859; 2024-11-24T16:42:52,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466572381, max timestamp=1732466572431Num uniques 859; 2024-11-24T16:42:52,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:52,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:52,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466572381, max timestamp=1732466572431Num uniques 859; 2024-11-24T16:42:52,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:42:52,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466632511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466632516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51342 deadline: 1732466632534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,547 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51374 deadline: 1732466632554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,569 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T16:42:52,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466632624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466632634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,674 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/17ee3444e5564a208e070c2b94d103dd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/17ee3444e5564a208e070c2b94d103dd 2024-11-24T16:42:52,677 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/A of 10af3cc4e904fab18c276c01940de289 into 17ee3444e5564a208e070c2b94d103dd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:52,677 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:52,677 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/A, priority=13, startTime=1732466572247; duration=0sec 2024-11-24T16:42:52,678 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:42:52,678 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:A 2024-11-24T16:42:52,678 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:42:52,678 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:42:52,678 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 10af3cc4e904fab18c276c01940de289/C is initiating minor compaction (all files) 2024-11-24T16:42:52,679 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 10af3cc4e904fab18c276c01940de289/C in TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,679 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/104e4ed47a8741cebd9a995dea63f782, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7ee7511abe4d4ce3819e176cc328b363, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8eedf671081f49db870af10820fa03eb] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp, totalSize=34.3 K 2024-11-24T16:42:52,679 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 104e4ed47a8741cebd9a995dea63f782, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732466567377 2024-11-24T16:42:52,679 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee7511abe4d4ce3819e176cc328b363, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732466568126 2024-11-24T16:42:52,680 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eedf671081f49db870af10820fa03eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1732466569766 2024-11-24T16:42:52,680 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/f9efbf0f1bab4523b982250660669a86 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f9efbf0f1bab4523b982250660669a86 2024-11-24T16:42:52,685 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/B of 10af3cc4e904fab18c276c01940de289 into f9efbf0f1bab4523b982250660669a86(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:52,685 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:52,685 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/B, priority=13, startTime=1732466572247; duration=0sec 2024-11-24T16:42:52,686 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:52,686 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:B 2024-11-24T16:42:52,686 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 10af3cc4e904fab18c276c01940de289#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:42:52,687 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/2bafaec540df43e0a8d5701406ae2666 is 50, key is test_row_0/C:col10/1732466569807/Put/seqid=0 2024-11-24T16:42:52,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742373_1549 (size=13085) 2024-11-24T16:42:52,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/749bedcf087a43d8b33432f70ecad3cc 2024-11-24T16:42:52,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/b67fca2677654e1f88d7918dde27f53a is 50, key is test_row_0/B:col10/1732466571037/Put/seqid=0 2024-11-24T16:42:52,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742374_1550 (size=12301) 2024-11-24T16:42:52,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466632843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466632844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:52,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:52,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:52,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:52,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T16:42:53,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,094 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/2bafaec540df43e0a8d5701406ae2666 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/2bafaec540df43e0a8d5701406ae2666 2024-11-24T16:42:53,099 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 10af3cc4e904fab18c276c01940de289/C of 10af3cc4e904fab18c276c01940de289 into 2bafaec540df43e0a8d5701406ae2666(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:42:53,099 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:53,099 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., storeName=10af3cc4e904fab18c276c01940de289/C, priority=13, startTime=1732466572248; duration=0sec 2024-11-24T16:42:53,099 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:42:53,099 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 10af3cc4e904fab18c276c01940de289:C 2024-11-24T16:42:53,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/b67fca2677654e1f88d7918dde27f53a 2024-11-24T16:42:53,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51370 deadline: 1732466633164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51354 deadline: 1732466633164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:53,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/23760521f9ab419b828f09e0d2bd5ca3 is 50, key is test_row_0/C:col10/1732466571037/Put/seqid=0 2024-11-24T16:42:53,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742375_1551 (size=12301) 2024-11-24T16:42:53,188 INFO [Thread-2136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2136. 2024-11-24T16:42:53,188 DEBUG [Thread-2136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:53,189 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,189 DEBUG [Thread-2136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:53,193 INFO [Thread-2134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2134. 2024-11-24T16:42:53,193 DEBUG [Thread-2134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:53,193 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,193 DEBUG [Thread-2134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:53,193 INFO [Thread-2132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2132. 2024-11-24T16:42:53,193 DEBUG [Thread-2132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:53,193 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,194 DEBUG [Thread-2132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,194 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:53,201 INFO [Thread-2130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2130. 2024-11-24T16:42:53,201 DEBUG [Thread-2130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:53,201 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,201 DEBUG [Thread-2130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:53,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,205 INFO [Thread-2138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2138. 2024-11-24T16:42:53,205 DEBUG [Thread-2138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:53,205 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,205 DEBUG [Thread-2138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:53,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:42:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51348 deadline: 1732466633234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 2024-11-24T16:42:53,235 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 10af3cc4e904fab18c276c01940de289 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:42:53,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 , the old value is region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:53,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:42:53,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., hostname=e335232f7491,33607,1732466431346, seqNum=2 because the exception is null or not the one we care about 2024-11-24T16:42:53,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T16:42:53,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:42:53,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/23760521f9ab419b828f09e0d2bd5ca3 2024-11-24T16:42:53,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/749bedcf087a43d8b33432f70ecad3cc as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/749bedcf087a43d8b33432f70ecad3cc 2024-11-24T16:42:53,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/749bedcf087a43d8b33432f70ecad3cc, entries=200, sequenceid=446, filesize=14.4 K 2024-11-24T16:42:53,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/b67fca2677654e1f88d7918dde27f53a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/b67fca2677654e1f88d7918dde27f53a 2024-11-24T16:42:53,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/b67fca2677654e1f88d7918dde27f53a, entries=150, sequenceid=446, filesize=12.0 K 2024-11-24T16:42:53,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/23760521f9ab419b828f09e0d2bd5ca3 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/23760521f9ab419b828f09e0d2bd5ca3 2024-11-24T16:42:53,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/23760521f9ab419b828f09e0d2bd5ca3, entries=150, sequenceid=446, filesize=12.0 K 2024-11-24T16:42:53,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 10af3cc4e904fab18c276c01940de289 in 1242ms, sequenceid=446, compaction requested=false 2024-11-24T16:42:53,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:53,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T16:42:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:53,662 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T16:42:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:42:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:42:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:42:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:42:53,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3443e07d6d4b41d382c101037815e732 is 50, key is test_row_0/A:col10/1732466572492/Put/seqid=0 2024-11-24T16:42:53,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742376_1552 (size=12301) 2024-11-24T16:42:53,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 10af3cc4e904fab18c276c01940de289 2024-11-24T16:42:53,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. as already flushing 2024-11-24T16:42:53,685 INFO [Thread-2119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2119. 2024-11-24T16:42:53,685 DEBUG [Thread-2119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:53,685 INFO [Thread-2125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2125. 2024-11-24T16:42:53,685 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,685 DEBUG [Thread-2125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:53,685 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,685 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:53,685 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:53,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:54,069 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3443e07d6d4b41d382c101037815e732 2024-11-24T16:42:54,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/37c4b85473cd4413b433936bf26bf386 is 50, key is test_row_0/B:col10/1732466572492/Put/seqid=0 2024-11-24T16:42:54,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742377_1553 (size=12301) 2024-11-24T16:42:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T16:42:54,477 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/37c4b85473cd4413b433936bf26bf386 2024-11-24T16:42:54,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/492847ff04214660a08930fb2651eb48 is 50, key is test_row_0/C:col10/1732466572492/Put/seqid=0 2024-11-24T16:42:54,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742378_1554 (size=12301) 2024-11-24T16:42:54,886 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/492847ff04214660a08930fb2651eb48 2024-11-24T16:42:54,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/3443e07d6d4b41d382c101037815e732 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3443e07d6d4b41d382c101037815e732 2024-11-24T16:42:54,892 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3443e07d6d4b41d382c101037815e732, entries=150, sequenceid=485, filesize=12.0 K 2024-11-24T16:42:54,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/37c4b85473cd4413b433936bf26bf386 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/37c4b85473cd4413b433936bf26bf386 2024-11-24T16:42:54,895 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/37c4b85473cd4413b433936bf26bf386, entries=150, sequenceid=485, filesize=12.0 K 2024-11-24T16:42:54,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/492847ff04214660a08930fb2651eb48 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/492847ff04214660a08930fb2651eb48 2024-11-24T16:42:54,898 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/492847ff04214660a08930fb2651eb48, entries=150, sequenceid=485, filesize=12.0 K 2024-11-24T16:42:54,899 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=13.42 KB/13740 for 10af3cc4e904fab18c276c01940de289 in 1237ms, sequenceid=485, compaction requested=true 2024-11-24T16:42:54,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 10af3cc4e904fab18c276c01940de289: 2024-11-24T16:42:54,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:42:54,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-24T16:42:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-24T16:42:54,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-24T16:42:54,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6130 sec 2024-11-24T16:42:54,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.6170 sec 2024-11-24T16:42:55,264 INFO [Thread-2127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2127. 2024-11-24T16:42:55,264 DEBUG [Thread-2127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:42:55,264 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:55,265 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:42:55,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:42:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T16:42:56,464 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:42:59,336 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T16:43:02,566 INFO [Thread-2123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2123. 2024-11-24T16:43:02,566 DEBUG [Thread-2123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:02,566 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:02,566 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:02,566 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:02,635 INFO [Thread-2121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2121. 2024-11-24T16:43:02,635 DEBUG [Thread-2121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:02,635 DEBUG [Thread-2121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:02,635 DEBUG [Thread-2121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:02,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1031 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3092 rows 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1031 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3093 rows 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1024 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3070 rows 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1028 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3084 rows 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1032 2024-11-24T16:43:02,635 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3091 rows 2024-11-24T16:43:02,636 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:43:02,636 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:43:02,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:02,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:02,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:02,636 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T16:43:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T16:43:02,639 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466582639"}]},"ts":"1732466582639"} 2024-11-24T16:43:02,640 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T16:43:02,640 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T16:43:02,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:43:02,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, UNASSIGN}] 2024-11-24T16:43:02,643 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, UNASSIGN 2024-11-24T16:43:02,643 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=10af3cc4e904fab18c276c01940de289, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:43:02,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, UNASSIGN because future has completed 2024-11-24T16:43:02,645 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:43:02,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:43:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T16:43:02,797 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 10af3cc4e904fab18c276c01940de289 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 10af3cc4e904fab18c276c01940de289, disabling compactions & flushes 2024-11-24T16:43:02,797 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. after waiting 0 ms 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:43:02,797 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 10af3cc4e904fab18c276c01940de289 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=A 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=B 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 10af3cc4e904fab18c276c01940de289, store=C 2024-11-24T16:43:02,797 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:02,801 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/a76a4218c2434db18af0737ba4f26078 is 50, key is test_row_0/A:col10/1732466575263/Put/seqid=0 2024-11-24T16:43:02,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742379_1555 (size=9857) 2024-11-24T16:43:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T16:43:03,204 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/a76a4218c2434db18af0737ba4f26078 2024-11-24T16:43:03,210 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/48bd2a72e7b1469c9d95c94413e0fce8 is 50, key is test_row_0/B:col10/1732466575263/Put/seqid=0 2024-11-24T16:43:03,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742380_1556 (size=9857) 2024-11-24T16:43:03,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T16:43:03,614 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/48bd2a72e7b1469c9d95c94413e0fce8 2024-11-24T16:43:03,620 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/2b3fce6bafa8476cb6c766b35eb47929 is 50, key is test_row_0/C:col10/1732466575263/Put/seqid=0 2024-11-24T16:43:03,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742381_1557 (size=9857) 2024-11-24T16:43:03,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T16:43:04,024 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/2b3fce6bafa8476cb6c766b35eb47929 2024-11-24T16:43:04,027 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/A/a76a4218c2434db18af0737ba4f26078 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a76a4218c2434db18af0737ba4f26078 2024-11-24T16:43:04,030 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a76a4218c2434db18af0737ba4f26078, entries=100, sequenceid=493, filesize=9.6 K 2024-11-24T16:43:04,030 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/B/48bd2a72e7b1469c9d95c94413e0fce8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/48bd2a72e7b1469c9d95c94413e0fce8 2024-11-24T16:43:04,033 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/48bd2a72e7b1469c9d95c94413e0fce8, entries=100, sequenceid=493, filesize=9.6 K 2024-11-24T16:43:04,034 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/.tmp/C/2b3fce6bafa8476cb6c766b35eb47929 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/2b3fce6bafa8476cb6c766b35eb47929 2024-11-24T16:43:04,036 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/2b3fce6bafa8476cb6c766b35eb47929, entries=100, sequenceid=493, filesize=9.6 K 2024-11-24T16:43:04,037 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 10af3cc4e904fab18c276c01940de289 in 1240ms, sequenceid=493, compaction requested=true 2024-11-24T16:43:04,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/024dd6a0e9c14eb3b13a7649b81dec22, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/e7408797e24e46e188ae90411e8589ea, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/f60b7d791a1b40438bf59f1ab5b3d3b4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5770df3f1b0240c59763b96f54a97be7, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d4b4937823c4faaa3e3961e8373e99f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/b6439c796b8f4082baa9f9598b15cf64, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4942636598374e778fec843d8c85e933, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3156af4050554c679bf918802cadcc36, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5b338068caad441ca6a15f0b10ffdf3d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4a5799240ba0400ea480424a14d0a21c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/c77c72a7495b4d07a0a92146b699a114, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/19967ba3eb9d4b5ba3c9791ff96839bf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/8546caa1093d4269b1d86a041458af3f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/0ddfe0350573470987bda22ad1525b56, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/876e4a6382e749db8b86c10c704aa805, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/72b46652703b40d4ae083c9d4f0bd0df, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/179f72114c414a36bad6aae404f57e25, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/88be0e4649ff461c863a58cd2c2b5826, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a1ffefb52b944dc493e7e4058236343c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/1f228f3211fc47b7987c575e0fc4273c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/524959d0329248f3b4c0341b0ae25553, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/81d1552087434b489cdc904452e3b79f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d719b4b942a4db59bca4acd8f7b82bb] to archive 2024-11-24T16:43:04,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:43:04,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/024dd6a0e9c14eb3b13a7649b81dec22 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/024dd6a0e9c14eb3b13a7649b81dec22 2024-11-24T16:43:04,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/e7408797e24e46e188ae90411e8589ea to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/e7408797e24e46e188ae90411e8589ea 2024-11-24T16:43:04,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/f60b7d791a1b40438bf59f1ab5b3d3b4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/f60b7d791a1b40438bf59f1ab5b3d3b4 2024-11-24T16:43:04,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5770df3f1b0240c59763b96f54a97be7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5770df3f1b0240c59763b96f54a97be7 2024-11-24T16:43:04,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d4b4937823c4faaa3e3961e8373e99f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d4b4937823c4faaa3e3961e8373e99f 2024-11-24T16:43:04,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/b6439c796b8f4082baa9f9598b15cf64 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/b6439c796b8f4082baa9f9598b15cf64 2024-11-24T16:43:04,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4942636598374e778fec843d8c85e933 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4942636598374e778fec843d8c85e933 2024-11-24T16:43:04,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3156af4050554c679bf918802cadcc36 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3156af4050554c679bf918802cadcc36 2024-11-24T16:43:04,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5b338068caad441ca6a15f0b10ffdf3d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/5b338068caad441ca6a15f0b10ffdf3d 2024-11-24T16:43:04,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4a5799240ba0400ea480424a14d0a21c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/4a5799240ba0400ea480424a14d0a21c 2024-11-24T16:43:04,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/c77c72a7495b4d07a0a92146b699a114 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/c77c72a7495b4d07a0a92146b699a114 2024-11-24T16:43:04,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/19967ba3eb9d4b5ba3c9791ff96839bf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/19967ba3eb9d4b5ba3c9791ff96839bf 2024-11-24T16:43:04,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/8546caa1093d4269b1d86a041458af3f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/8546caa1093d4269b1d86a041458af3f 2024-11-24T16:43:04,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/0ddfe0350573470987bda22ad1525b56 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/0ddfe0350573470987bda22ad1525b56 2024-11-24T16:43:04,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/876e4a6382e749db8b86c10c704aa805 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/876e4a6382e749db8b86c10c704aa805 2024-11-24T16:43:04,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/72b46652703b40d4ae083c9d4f0bd0df to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/72b46652703b40d4ae083c9d4f0bd0df 2024-11-24T16:43:04,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/179f72114c414a36bad6aae404f57e25 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/179f72114c414a36bad6aae404f57e25 2024-11-24T16:43:04,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/88be0e4649ff461c863a58cd2c2b5826 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/88be0e4649ff461c863a58cd2c2b5826 2024-11-24T16:43:04,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a1ffefb52b944dc493e7e4058236343c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a1ffefb52b944dc493e7e4058236343c 2024-11-24T16:43:04,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/1f228f3211fc47b7987c575e0fc4273c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/1f228f3211fc47b7987c575e0fc4273c 2024-11-24T16:43:04,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/524959d0329248f3b4c0341b0ae25553 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/524959d0329248f3b4c0341b0ae25553 2024-11-24T16:43:04,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/81d1552087434b489cdc904452e3b79f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/81d1552087434b489cdc904452e3b79f 2024-11-24T16:43:04,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d719b4b942a4db59bca4acd8f7b82bb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3d719b4b942a4db59bca4acd8f7b82bb 2024-11-24T16:43:04,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/49175b970ab44e80876ef151d7c6e3bd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8fc85074ab884dd0954739c665952f7f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/c447e26923fb4ec29328fc0a99787842, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8eb7d839c08a4be2953afd08e6708876, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bc38899d3aba490b844f023a2ebae7fd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1841245877af4e31ae016cc5e20e1dff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/d4bc36ffd019442e803e862b542f475f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/aa552c76ba2744d3a2efd97ff0b5bb1f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/220763793f6c4a4ca7bd6d67b7f87c22, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/0310d80522ff4402bddb71ff905a333a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/5149bc9eb47143398c274e7ac30584a0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3e7a89611e754bc1a516b466946405cf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bf24b9e807564a35b88f3d81a1b70240, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1d664bbb10524040a87b5eabee3d5e15, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/44f93c6fc58641c9be5bcb4e5afdc391, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bd825d54895840c5997ea8a3639e4625, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/7538282e0a9b453cb58710949cab0b79, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1a03ca0fd83643d39f1c1658bf0ff79b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/81f004877d134ee39f6a1dfa204c60df, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/ba5b3255c030488ca91ff9393b905c44, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3d3d177d5f1f4d6db547df53cd7cd1a9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f283b2c29cfa42b19358ce1d9081b55d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/660da995f48440d7818ca6834de18efc] to archive 2024-11-24T16:43:04,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:43:04,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/49175b970ab44e80876ef151d7c6e3bd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/49175b970ab44e80876ef151d7c6e3bd 2024-11-24T16:43:04,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8fc85074ab884dd0954739c665952f7f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8fc85074ab884dd0954739c665952f7f 2024-11-24T16:43:04,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/c447e26923fb4ec29328fc0a99787842 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/c447e26923fb4ec29328fc0a99787842 2024-11-24T16:43:04,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8eb7d839c08a4be2953afd08e6708876 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/8eb7d839c08a4be2953afd08e6708876 2024-11-24T16:43:04,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bc38899d3aba490b844f023a2ebae7fd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bc38899d3aba490b844f023a2ebae7fd 2024-11-24T16:43:04,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1841245877af4e31ae016cc5e20e1dff to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1841245877af4e31ae016cc5e20e1dff 2024-11-24T16:43:04,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/d4bc36ffd019442e803e862b542f475f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/d4bc36ffd019442e803e862b542f475f 2024-11-24T16:43:04,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/aa552c76ba2744d3a2efd97ff0b5bb1f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/aa552c76ba2744d3a2efd97ff0b5bb1f 2024-11-24T16:43:04,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/220763793f6c4a4ca7bd6d67b7f87c22 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/220763793f6c4a4ca7bd6d67b7f87c22 2024-11-24T16:43:04,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/0310d80522ff4402bddb71ff905a333a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/0310d80522ff4402bddb71ff905a333a 2024-11-24T16:43:04,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/5149bc9eb47143398c274e7ac30584a0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/5149bc9eb47143398c274e7ac30584a0 2024-11-24T16:43:04,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3e7a89611e754bc1a516b466946405cf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3e7a89611e754bc1a516b466946405cf 2024-11-24T16:43:04,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bf24b9e807564a35b88f3d81a1b70240 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bf24b9e807564a35b88f3d81a1b70240 2024-11-24T16:43:04,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1d664bbb10524040a87b5eabee3d5e15 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1d664bbb10524040a87b5eabee3d5e15 2024-11-24T16:43:04,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/44f93c6fc58641c9be5bcb4e5afdc391 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/44f93c6fc58641c9be5bcb4e5afdc391 2024-11-24T16:43:04,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bd825d54895840c5997ea8a3639e4625 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/bd825d54895840c5997ea8a3639e4625 2024-11-24T16:43:04,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/7538282e0a9b453cb58710949cab0b79 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/7538282e0a9b453cb58710949cab0b79 2024-11-24T16:43:04,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1a03ca0fd83643d39f1c1658bf0ff79b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/1a03ca0fd83643d39f1c1658bf0ff79b 2024-11-24T16:43:04,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/81f004877d134ee39f6a1dfa204c60df to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/81f004877d134ee39f6a1dfa204c60df 2024-11-24T16:43:04,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/ba5b3255c030488ca91ff9393b905c44 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/ba5b3255c030488ca91ff9393b905c44 2024-11-24T16:43:04,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3d3d177d5f1f4d6db547df53cd7cd1a9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/3d3d177d5f1f4d6db547df53cd7cd1a9 2024-11-24T16:43:04,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f283b2c29cfa42b19358ce1d9081b55d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f283b2c29cfa42b19358ce1d9081b55d 2024-11-24T16:43:04,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/660da995f48440d7818ca6834de18efc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/660da995f48440d7818ca6834de18efc 2024-11-24T16:43:04,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/4df72c8204bf46248fe1f65c57c2e565, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f24e7f7ff58c42569068e6e073afd1d0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/ae3714240591418cb0e01de0e6a9f380, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8c430ad06a904c67bd3152ce6af1b6f6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/a23b66fba6a948259f4fd2fdb8890401, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/bc133f6879704edab5c0540bac73c8d0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/67184210b601403a89252beab5f212ed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/25a1a467ef1240d4a022bf0604d8ed47, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/640c18e28649434d8d690ab3b8ec0f2a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7fd696bad0644740b4d9fbb9428c5be0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1ad912106aeb4f9fa80d7d26a952eea3, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f444a1a33eb04c63bb6c84b77fa7aa35, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/d29391bea2a04dcc92b20a290b3fa829, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/42df7d1b5c83418998b613370ff86b69, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/9ec81744ce194c85b3e991756688bf03, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/38b9782533e34151bcdfc844dca5e27f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1b5dc2f9f32345b18988f9a9851bbeda, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/5d6de0e8a402433c9309f75ac3e6892d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/6bc10ab6e1494de58eb3d9d00c7d9184, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/104e4ed47a8741cebd9a995dea63f782, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1cdb2d532e0d4d39b5207b8770baace6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7ee7511abe4d4ce3819e176cc328b363, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8eedf671081f49db870af10820fa03eb] to archive 2024-11-24T16:43:04,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:43:04,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/4df72c8204bf46248fe1f65c57c2e565 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/4df72c8204bf46248fe1f65c57c2e565 2024-11-24T16:43:04,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f24e7f7ff58c42569068e6e073afd1d0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f24e7f7ff58c42569068e6e073afd1d0 2024-11-24T16:43:04,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/ae3714240591418cb0e01de0e6a9f380 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/ae3714240591418cb0e01de0e6a9f380 2024-11-24T16:43:04,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8c430ad06a904c67bd3152ce6af1b6f6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8c430ad06a904c67bd3152ce6af1b6f6 2024-11-24T16:43:04,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/a23b66fba6a948259f4fd2fdb8890401 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/a23b66fba6a948259f4fd2fdb8890401 2024-11-24T16:43:04,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/bc133f6879704edab5c0540bac73c8d0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/bc133f6879704edab5c0540bac73c8d0 2024-11-24T16:43:04,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/67184210b601403a89252beab5f212ed to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/67184210b601403a89252beab5f212ed 2024-11-24T16:43:04,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/25a1a467ef1240d4a022bf0604d8ed47 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/25a1a467ef1240d4a022bf0604d8ed47 2024-11-24T16:43:04,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/640c18e28649434d8d690ab3b8ec0f2a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/640c18e28649434d8d690ab3b8ec0f2a 2024-11-24T16:43:04,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7fd696bad0644740b4d9fbb9428c5be0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7fd696bad0644740b4d9fbb9428c5be0 2024-11-24T16:43:04,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1ad912106aeb4f9fa80d7d26a952eea3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1ad912106aeb4f9fa80d7d26a952eea3 2024-11-24T16:43:04,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f444a1a33eb04c63bb6c84b77fa7aa35 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/f444a1a33eb04c63bb6c84b77fa7aa35 2024-11-24T16:43:04,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/d29391bea2a04dcc92b20a290b3fa829 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/d29391bea2a04dcc92b20a290b3fa829 2024-11-24T16:43:04,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/42df7d1b5c83418998b613370ff86b69 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/42df7d1b5c83418998b613370ff86b69 2024-11-24T16:43:04,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/9ec81744ce194c85b3e991756688bf03 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/9ec81744ce194c85b3e991756688bf03 2024-11-24T16:43:04,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/38b9782533e34151bcdfc844dca5e27f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/38b9782533e34151bcdfc844dca5e27f 2024-11-24T16:43:04,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1b5dc2f9f32345b18988f9a9851bbeda to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1b5dc2f9f32345b18988f9a9851bbeda 2024-11-24T16:43:04,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/5d6de0e8a402433c9309f75ac3e6892d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/5d6de0e8a402433c9309f75ac3e6892d 2024-11-24T16:43:04,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/6bc10ab6e1494de58eb3d9d00c7d9184 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/6bc10ab6e1494de58eb3d9d00c7d9184 2024-11-24T16:43:04,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/104e4ed47a8741cebd9a995dea63f782 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/104e4ed47a8741cebd9a995dea63f782 2024-11-24T16:43:04,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1cdb2d532e0d4d39b5207b8770baace6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/1cdb2d532e0d4d39b5207b8770baace6 2024-11-24T16:43:04,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7ee7511abe4d4ce3819e176cc328b363 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/7ee7511abe4d4ce3819e176cc328b363 2024-11-24T16:43:04,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8eedf671081f49db870af10820fa03eb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/8eedf671081f49db870af10820fa03eb 2024-11-24T16:43:04,103 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/recovered.edits/496.seqid, newMaxSeqId=496, maxSeqId=1 2024-11-24T16:43:04,104 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. 2024-11-24T16:43:04,104 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 10af3cc4e904fab18c276c01940de289: Waiting for close lock at 1732466582797Running coprocessor pre-close hooks at 1732466582797Disabling compacts and flushes for region at 1732466582797Disabling writes for close at 1732466582797Obtaining lock to block concurrent updates at 1732466582797Preparing flush snapshotting stores in 10af3cc4e904fab18c276c01940de289 at 1732466582797Finished memstore snapshotting TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732466582797Flushing stores of TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289. at 1732466582798 (+1 ms)Flushing 10af3cc4e904fab18c276c01940de289/A: creating writer at 1732466582798Flushing 10af3cc4e904fab18c276c01940de289/A: appending metadata at 1732466582800 (+2 ms)Flushing 10af3cc4e904fab18c276c01940de289/A: closing flushed file at 1732466582800Flushing 10af3cc4e904fab18c276c01940de289/B: creating writer at 1732466583208 (+408 ms)Flushing 10af3cc4e904fab18c276c01940de289/B: appending metadata at 1732466583210 (+2 ms)Flushing 10af3cc4e904fab18c276c01940de289/B: closing flushed file at 1732466583210Flushing 10af3cc4e904fab18c276c01940de289/C: creating writer at 1732466583617 (+407 ms)Flushing 10af3cc4e904fab18c276c01940de289/C: appending metadata at 1732466583620 (+3 ms)Flushing 10af3cc4e904fab18c276c01940de289/C: closing flushed file at 1732466583620Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ab9e7f8: reopening flushed file at 1732466584027 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cad44bb: reopening flushed file at 1732466584030 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36301a5b: reopening flushed file at 1732466584033 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 10af3cc4e904fab18c276c01940de289 in 1240ms, sequenceid=493, compaction requested=true at 1732466584037 (+4 ms)Writing region close event to WAL at 1732466584101 (+64 ms)Running coprocessor post-close hooks at 1732466584104 (+3 ms)Closed at 1732466584104 2024-11-24T16:43:04,106 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 10af3cc4e904fab18c276c01940de289 2024-11-24T16:43:04,106 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=10af3cc4e904fab18c276c01940de289, regionState=CLOSED 2024-11-24T16:43:04,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:43:04,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-24T16:43:04,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 10af3cc4e904fab18c276c01940de289, server=e335232f7491,33607,1732466431346 in 1.4640 sec 2024-11-24T16:43:04,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-24T16:43:04,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=10af3cc4e904fab18c276c01940de289, UNASSIGN in 1.4690 sec 2024-11-24T16:43:04,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-24T16:43:04,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4720 sec 2024-11-24T16:43:04,115 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466584115"}]},"ts":"1732466584115"} 2024-11-24T16:43:04,117 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T16:43:04,117 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T16:43:04,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-24T16:43:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T16:43:04,784 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:04,785 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T16:43:04,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:04,786 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:04,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T16:43:04,787 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:04,789 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289 2024-11-24T16:43:04,790 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/recovered.edits] 2024-11-24T16:43:04,792 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/17ee3444e5564a208e070c2b94d103dd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/17ee3444e5564a208e070c2b94d103dd 2024-11-24T16:43:04,793 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3443e07d6d4b41d382c101037815e732 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/3443e07d6d4b41d382c101037815e732 2024-11-24T16:43:04,794 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/749bedcf087a43d8b33432f70ecad3cc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/749bedcf087a43d8b33432f70ecad3cc 2024-11-24T16:43:04,794 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a76a4218c2434db18af0737ba4f26078 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/A/a76a4218c2434db18af0737ba4f26078 2024-11-24T16:43:04,796 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/37c4b85473cd4413b433936bf26bf386 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/37c4b85473cd4413b433936bf26bf386 2024-11-24T16:43:04,796 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/48bd2a72e7b1469c9d95c94413e0fce8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/48bd2a72e7b1469c9d95c94413e0fce8 2024-11-24T16:43:04,797 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/b67fca2677654e1f88d7918dde27f53a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/b67fca2677654e1f88d7918dde27f53a 2024-11-24T16:43:04,798 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f9efbf0f1bab4523b982250660669a86 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/B/f9efbf0f1bab4523b982250660669a86 2024-11-24T16:43:04,800 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/23760521f9ab419b828f09e0d2bd5ca3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/23760521f9ab419b828f09e0d2bd5ca3 2024-11-24T16:43:04,800 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/2b3fce6bafa8476cb6c766b35eb47929 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/2b3fce6bafa8476cb6c766b35eb47929 2024-11-24T16:43:04,801 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/2bafaec540df43e0a8d5701406ae2666 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/2bafaec540df43e0a8d5701406ae2666 2024-11-24T16:43:04,802 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/492847ff04214660a08930fb2651eb48 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/C/492847ff04214660a08930fb2651eb48 2024-11-24T16:43:04,803 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/recovered.edits/496.seqid to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289/recovered.edits/496.seqid 2024-11-24T16:43:04,804 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/10af3cc4e904fab18c276c01940de289 2024-11-24T16:43:04,804 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T16:43:04,805 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:04,807 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T16:43:04,808 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T16:43:04,809 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:04,809 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T16:43:04,809 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732466584809"}]},"ts":"9223372036854775807"} 2024-11-24T16:43:04,811 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T16:43:04,811 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 10af3cc4e904fab18c276c01940de289, NAME => 'TestAcidGuarantees,,1732466550991.10af3cc4e904fab18c276c01940de289.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T16:43:04,811 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T16:43:04,811 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732466584811"}]},"ts":"9223372036854775807"} 2024-11-24T16:43:04,812 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T16:43:04,813 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-24T16:43:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T16:43:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T16:43:04,897 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:04,903 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=244 (was 245), OpenFileDescriptor=441 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=380 (was 461), ProcessCount=11 (was 11), AvailableMemoryMB=5753 (was 5841) 2024-11-24T16:43:04,911 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=244, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=380, ProcessCount=11, AvailableMemoryMB=5752 2024-11-24T16:43:04,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:43:04,913 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:43:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:04,914 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T16:43:04,915 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:04,915 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-24T16:43:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T16:43:04,915 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T16:43:04,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742382_1558 (size=1061) 2024-11-24T16:43:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T16:43:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T16:43:05,321 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446 2024-11-24T16:43:05,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742383_1559 (size=53) 2024-11-24T16:43:05,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T16:43:05,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:43:05,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8e3281f5174968dfe4494f3e286572d3, disabling compactions & flushes 2024-11-24T16:43:05,726 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:05,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:05,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. after waiting 0 ms 2024-11-24T16:43:05,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:05,726 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:05,726 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8e3281f5174968dfe4494f3e286572d3: Waiting for close lock at 1732466585726Disabling compacts and flushes for region at 1732466585726Disabling writes for close at 1732466585726Writing region close event to WAL at 1732466585726Closed at 1732466585726 2024-11-24T16:43:05,727 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T16:43:05,728 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732466585727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732466585727"}]},"ts":"1732466585727"} 2024-11-24T16:43:05,729 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T16:43:05,729 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T16:43:05,730 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466585729"}]},"ts":"1732466585729"} 2024-11-24T16:43:05,730 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T16:43:05,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, ASSIGN}] 2024-11-24T16:43:05,731 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, ASSIGN 2024-11-24T16:43:05,732 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, ASSIGN; state=OFFLINE, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=false 2024-11-24T16:43:05,883 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:43:05,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, ASSIGN because future has completed 2024-11-24T16:43:05,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:43:06,038 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:06,039 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:43:06,039 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,039 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:43:06,039 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,039 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,040 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,041 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:43:06,041 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e3281f5174968dfe4494f3e286572d3 columnFamilyName A 2024-11-24T16:43:06,041 DEBUG [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:06,041 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(327): Store=8e3281f5174968dfe4494f3e286572d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:43:06,042 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,042 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:43:06,042 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e3281f5174968dfe4494f3e286572d3 columnFamilyName B 2024-11-24T16:43:06,042 DEBUG [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:06,043 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(327): Store=8e3281f5174968dfe4494f3e286572d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:43:06,043 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,043 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:43:06,043 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e3281f5174968dfe4494f3e286572d3 columnFamilyName C 2024-11-24T16:43:06,043 DEBUG [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:06,044 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(327): Store=8e3281f5174968dfe4494f3e286572d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:43:06,044 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:06,044 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,044 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,045 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,045 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,045 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,046 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:43:06,047 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,048 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T16:43:06,048 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 8e3281f5174968dfe4494f3e286572d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69689677, jitterRate=0.03845711052417755}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:43:06,048 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:06,049 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 8e3281f5174968dfe4494f3e286572d3: Running coprocessor pre-open hook at 1732466586039Writing region info on filesystem at 1732466586039Initializing all the Stores at 1732466586039Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466586040 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466586040Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466586040Cleaning up temporary data from old regions at 1732466586045 (+5 ms)Running coprocessor post-open hooks at 1732466586048 (+3 ms)Region opened successfully at 1732466586049 (+1 ms) 2024-11-24T16:43:06,049 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., pid=146, masterSystemTime=1732466586036 2024-11-24T16:43:06,051 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:06,051 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:06,051 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=OPEN, openSeqNum=2, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:43:06,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:43:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T16:43:06,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-24T16:43:06,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 in 168 msec 2024-11-24T16:43:06,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-24T16:43:06,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, ASSIGN in 323 msec 2024-11-24T16:43:06,056 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T16:43:06,056 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466586056"}]},"ts":"1732466586056"} 2024-11-24T16:43:06,057 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T16:43:06,058 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T16:43:06,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-24T16:43:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-24T16:43:07,064 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:07,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cec06c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:07,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:07,066 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:07,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dc3e29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:07,068 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e4443c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:07,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:07,071 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T16:43:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e335232f7491,37979,1732466430336 2024-11-24T16:43:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1cddbb80 2024-11-24T16:43:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T16:43:07,072 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T16:43:07,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T16:43:07,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T16:43:07,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:07,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742384_1560 (size=1097) 2024-11-24T16:43:07,483 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T16:43:07,483 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T16:43:07,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:43:07,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, REOPEN/MOVE}] 2024-11-24T16:43:07,487 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, REOPEN/MOVE 2024-11-24T16:43:07,488 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:43:07,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, REOPEN/MOVE because future has completed 2024-11-24T16:43:07,489 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:43:07,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:43:07,641 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,641 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:43:07,641 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 8e3281f5174968dfe4494f3e286572d3, disabling compactions & flushes 2024-11-24T16:43:07,641 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,641 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,641 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. after waiting 0 ms 2024-11-24T16:43:07,641 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,644 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T16:43:07,645 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,645 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 8e3281f5174968dfe4494f3e286572d3: Waiting for close lock at 1732466587641Running coprocessor pre-close hooks at 1732466587641Disabling compacts and flushes for region at 1732466587641Disabling writes for close at 1732466587641Writing region close event to WAL at 1732466587642 (+1 ms)Running coprocessor post-close hooks at 1732466587645 (+3 ms)Closed at 1732466587645 2024-11-24T16:43:07,645 WARN [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 8e3281f5174968dfe4494f3e286572d3 to self. 2024-11-24T16:43:07,646 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,647 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=CLOSED 2024-11-24T16:43:07,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:43:07,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-24T16:43:07,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 in 160 msec 2024-11-24T16:43:07,650 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, REOPEN/MOVE; state=CLOSED, location=e335232f7491,33607,1732466431346; forceNewPlan=false, retain=true 2024-11-24T16:43:07,801 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=OPENING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:43:07,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, REOPEN/MOVE because future has completed 2024-11-24T16:43:07,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:43:07,959 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,959 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T16:43:07,959 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,959 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T16:43:07,959 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,959 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,965 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,966 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:43:07,966 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e3281f5174968dfe4494f3e286572d3 columnFamilyName A 2024-11-24T16:43:07,967 DEBUG [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:07,968 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(327): Store=8e3281f5174968dfe4494f3e286572d3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:43:07,968 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,969 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:43:07,969 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e3281f5174968dfe4494f3e286572d3 columnFamilyName B 2024-11-24T16:43:07,969 DEBUG [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:07,969 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(327): Store=8e3281f5174968dfe4494f3e286572d3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:43:07,970 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,970 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T16:43:07,970 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e3281f5174968dfe4494f3e286572d3 columnFamilyName C 2024-11-24T16:43:07,970 DEBUG [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:07,971 INFO [StoreOpener-8e3281f5174968dfe4494f3e286572d3-1 {}] regionserver.HStore(327): Store=8e3281f5174968dfe4494f3e286572d3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T16:43:07,971 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,971 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,972 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,972 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,973 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,973 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,974 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T16:43:07,975 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,975 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 8e3281f5174968dfe4494f3e286572d3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64800497, jitterRate=-0.03439734876155853}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T16:43:07,975 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:07,976 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 8e3281f5174968dfe4494f3e286572d3: Running coprocessor pre-open hook at 1732466587959Writing region info on filesystem at 1732466587959Initializing all the Stores at 1732466587965 (+6 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466587965Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466587965Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732466587965Cleaning up temporary data from old regions at 1732466587973 (+8 ms)Running coprocessor post-open hooks at 1732466587975 (+2 ms)Region opened successfully at 1732466587976 (+1 ms) 2024-11-24T16:43:07,976 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., pid=151, masterSystemTime=1732466587955 2024-11-24T16:43:07,978 DEBUG [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,978 INFO [RS_OPEN_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:07,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=OPEN, openSeqNum=5, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:43:07,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:43:07,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-24T16:43:07,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 in 178 msec 2024-11-24T16:43:07,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-24T16:43:07,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, REOPEN/MOVE in 495 msec 2024-11-24T16:43:07,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-24T16:43:07,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-24T16:43:07,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 914 msec 2024-11-24T16:43:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-24T16:43:07,990 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:07,991 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:07,991 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:07,992 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320a16b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:07,993 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:07,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cdf6249, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:07,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:07,995 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3693b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:07,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:07,997 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:07,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516baa62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6604d96b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,008 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223d107e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,010 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e95dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d07bb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,014 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18492591, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db93f8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,017 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1c77f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,017 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fae240f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25605fac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c58caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,023 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3ba279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,024 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5176dc37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,026 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7a6e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,027 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed295ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e335232f7491,37979,-1 for getting cluster id 2024-11-24T16:43:08,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T16:43:08,029 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '83e3945b-9878-4aae-ad46-2020942a459f' 2024-11-24T16:43:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T16:43:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "83e3945b-9878-4aae-ad46-2020942a459f" 2024-11-24T16:43:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388a31fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e335232f7491,37979,-1] 2024-11-24T16:43:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T16:43:08,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:08,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T16:43:08,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1222154d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T16:43:08,031 DEBUG [Thread-2490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,031 DEBUG [Thread-2488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,031 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,032 DEBUG [Thread-2499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,032 DEBUG [Thread-2492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,033 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,033 DEBUG [Thread-2505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:43:08,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-24T16:43:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T16:43:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,039 DEBUG [Thread-2499 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,039 DEBUG [Thread-2499 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:43:08,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:43:08,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:43:08,040 DEBUG [Thread-2503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,044 DEBUG [Thread-2501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,048 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T16:43:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e335232f7491,33607,1732466431346, seqNum=-1] 2024-11-24T16:43:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T16:43:08,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T16:43:08,052 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5] 2024-11-24T16:43:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:08,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T16:43:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:08,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:08,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f06acadaa2624f59b20ae1e7921471d9_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466588056/Put/seqid=0 2024-11-24T16:43:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466648102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466648109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466588095, max timestamp=1732466588111Num uniques 859; 2024-11-24T16:43:08,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:43:08,115 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466588095, max timestamp=1732466588111Num uniques 859; 2024-11-24T16:43:08,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:43:08,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466588095, max timestamp=1732466588111Num uniques 859; 2024-11-24T16:43:08,116 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:43:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742385_1561 (size=12154) 2024-11-24T16:43:08,117 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:08,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f06acadaa2624f59b20ae1e7921471d9_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f06acadaa2624f59b20ae1e7921471d9_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:08,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/db926aa5939b4f668c5eba4a69c15867, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:08,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/db926aa5939b4f668c5eba4a69c15867 is 175, key is test_row_0/A:col10/1732466588056/Put/seqid=0 2024-11-24T16:43:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T16:43:08,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742386_1562 (size=30955) 2024-11-24T16:43:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466648143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466648146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466648150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:08,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:08,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466648223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466648223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466648254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466648263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466648263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:08,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T16:43:08,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:08,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466648433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466648433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466648464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466648473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466648474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:08,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:08,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/db926aa5939b4f668c5eba4a69c15867 2024-11-24T16:43:08,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/1eb1f0ae1ac6456383be5a08580a704e is 50, key is test_row_0/B:col10/1732466588056/Put/seqid=0 2024-11-24T16:43:08,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742387_1563 (size=12001) 2024-11-24T16:43:08,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/1eb1f0ae1ac6456383be5a08580a704e 2024-11-24T16:43:08,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/608f426f1802480b9fe0fc30f35062b6 is 50, key is test_row_0/C:col10/1732466588056/Put/seqid=0 2024-11-24T16:43:08,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742388_1564 (size=12001) 2024-11-24T16:43:08,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/608f426f1802480b9fe0fc30f35062b6 2024-11-24T16:43:08,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/db926aa5939b4f668c5eba4a69c15867 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/db926aa5939b4f668c5eba4a69c15867 2024-11-24T16:43:08,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/db926aa5939b4f668c5eba4a69c15867, entries=150, sequenceid=16, filesize=30.2 K 2024-11-24T16:43:08,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/1eb1f0ae1ac6456383be5a08580a704e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1eb1f0ae1ac6456383be5a08580a704e 2024-11-24T16:43:08,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1eb1f0ae1ac6456383be5a08580a704e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T16:43:08,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/608f426f1802480b9fe0fc30f35062b6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/608f426f1802480b9fe0fc30f35062b6 2024-11-24T16:43:08,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/608f426f1802480b9fe0fc30f35062b6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T16:43:08,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 8e3281f5174968dfe4494f3e286572d3 in 585ms, sequenceid=16, compaction requested=false 2024-11-24T16:43:08,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T16:43:08,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:08,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-24T16:43:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:08,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:08,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d542b59a6c4f413b9afddf366b880ab5_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466588148/Put/seqid=0 2024-11-24T16:43:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466648774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466648774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466648774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742389_1565 (size=14594) 2024-11-24T16:43:08,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466648785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466648785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,789 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:08,792 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d542b59a6c4f413b9afddf366b880ab5_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d542b59a6c4f413b9afddf366b880ab5_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:08,793 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a9bc7afff8c549c6ad897c21f2ef94cb, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:08,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a9bc7afff8c549c6ad897c21f2ef94cb is 175, key is test_row_0/A:col10/1732466588148/Put/seqid=0 2024-11-24T16:43:08,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742390_1566 (size=39549) 2024-11-24T16:43:08,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a9bc7afff8c549c6ad897c21f2ef94cb 2024-11-24T16:43:08,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/e1cf10ee84234b96bd755d75d74f6589 is 50, key is test_row_0/B:col10/1732466588148/Put/seqid=0 2024-11-24T16:43:08,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:08,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:08,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742391_1567 (size=12001) 2024-11-24T16:43:08,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/e1cf10ee84234b96bd755d75d74f6589 2024-11-24T16:43:08,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/4dd8f85b6b7f4661beff8e15c0035e26 is 50, key is test_row_0/C:col10/1732466588148/Put/seqid=0 2024-11-24T16:43:08,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742392_1568 (size=12001) 2024-11-24T16:43:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466648884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466648884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:08,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:08,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:09,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466649094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466649094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:09,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:09,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T16:43:09,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/4dd8f85b6b7f4661beff8e15c0035e26 2024-11-24T16:43:09,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a9bc7afff8c549c6ad897c21f2ef94cb as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a9bc7afff8c549c6ad897c21f2ef94cb 2024-11-24T16:43:09,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a9bc7afff8c549c6ad897c21f2ef94cb, entries=200, sequenceid=52, filesize=38.6 K 2024-11-24T16:43:09,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/e1cf10ee84234b96bd755d75d74f6589 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e1cf10ee84234b96bd755d75d74f6589 2024-11-24T16:43:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e1cf10ee84234b96bd755d75d74f6589, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T16:43:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/4dd8f85b6b7f4661beff8e15c0035e26 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/4dd8f85b6b7f4661beff8e15c0035e26 2024-11-24T16:43:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/4dd8f85b6b7f4661beff8e15c0035e26, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T16:43:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 8e3281f5174968dfe4494f3e286572d3 in 525ms, sequenceid=52, compaction requested=false 2024-11-24T16:43:09,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-24T16:43:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,273 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:43:09,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:09,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e3c10c0daa9540708d07eb697d0fbbae_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466588772/Put/seqid=0 2024-11-24T16:43:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742393_1569 (size=9714) 2024-11-24T16:43:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466589346, max timestamp=1732466589355Num uniques 859; 2024-11-24T16:43:09,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732466589346, max timestamp=1732466589355Num uniques 859; 2024-11-24T16:43:09,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:43:09,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T16:43:09,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466589346, max timestamp=1732466589355Num uniques 859; 2024-11-24T16:43:09,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:09,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466649415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466649415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466649415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466649422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466649423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466649533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466649533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466649534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466649534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466649534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,626 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T16:43:09,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:09,722 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e3c10c0daa9540708d07eb697d0fbbae_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e3c10c0daa9540708d07eb697d0fbbae_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:09,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/540d037bc9fe43e3accddd55235aae6b, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:09,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/540d037bc9fe43e3accddd55235aae6b is 175, key is test_row_0/A:col10/1732466588772/Put/seqid=0 2024-11-24T16:43:09,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742394_1570 (size=22361) 2024-11-24T16:43:09,741 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/540d037bc9fe43e3accddd55235aae6b 2024-11-24T16:43:09,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466649744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466649744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466649744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466649744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466649744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:09,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:09,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/4e39aa5230134c4db03ec076d25fbbcd is 50, key is test_row_0/B:col10/1732466588772/Put/seqid=0 2024-11-24T16:43:09,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742395_1571 (size=9657) 2024-11-24T16:43:10,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466650054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466650054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466650054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466650054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466650054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,164 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/4e39aa5230134c4db03ec076d25fbbcd 2024-11-24T16:43:10,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/56919711ece140a99494481263a114ed is 50, key is test_row_0/C:col10/1732466588772/Put/seqid=0 2024-11-24T16:43:10,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742396_1572 (size=9657) 2024-11-24T16:43:10,180 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/56919711ece140a99494481263a114ed 2024-11-24T16:43:10,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/540d037bc9fe43e3accddd55235aae6b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/540d037bc9fe43e3accddd55235aae6b 2024-11-24T16:43:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T16:43:10,189 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/540d037bc9fe43e3accddd55235aae6b, entries=100, sequenceid=63, filesize=21.8 K 2024-11-24T16:43:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/4e39aa5230134c4db03ec076d25fbbcd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4e39aa5230134c4db03ec076d25fbbcd 2024-11-24T16:43:10,192 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4e39aa5230134c4db03ec076d25fbbcd, entries=100, sequenceid=63, filesize=9.4 K 2024-11-24T16:43:10,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/56919711ece140a99494481263a114ed as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/56919711ece140a99494481263a114ed 2024-11-24T16:43:10,199 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/56919711ece140a99494481263a114ed, entries=100, sequenceid=63, filesize=9.4 K 2024-11-24T16:43:10,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 8e3281f5174968dfe4494f3e286572d3 in 927ms, sequenceid=63, compaction requested=true 2024-11-24T16:43:10,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:10,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:10,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-24T16:43:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-24T16:43:10,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-24T16:43:10,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1620 sec 2024-11-24T16:43:10,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1690 sec 2024-11-24T16:43:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:10,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-24T16:43:10,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:10,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:10,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:10,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:10,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:10,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:10,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112411980183e09642baada18facc6ca8c6d_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466590565/Put/seqid=0 2024-11-24T16:43:10,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742397_1573 (size=14594) 2024-11-24T16:43:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466650575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466650575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466650576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466650576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466650581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466650693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466650694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466650694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466650694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466650903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466650903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466650903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466650904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:10,982 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:10,986 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112411980183e09642baada18facc6ca8c6d_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112411980183e09642baada18facc6ca8c6d_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:10,989 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/5b9e2ae4e9294376b1b364ffc57157be, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:10,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/5b9e2ae4e9294376b1b364ffc57157be is 175, key is test_row_0/A:col10/1732466590565/Put/seqid=0 2024-11-24T16:43:10,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742398_1574 (size=39549) 2024-11-24T16:43:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466651214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466651214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466651214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466651214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,393 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/5b9e2ae4e9294376b1b364ffc57157be 2024-11-24T16:43:11,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/aba337f956a1474a9c9c838946490352 is 50, key is test_row_0/B:col10/1732466590565/Put/seqid=0 2024-11-24T16:43:11,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742399_1575 (size=12001) 2024-11-24T16:43:11,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466651594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,596 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466651724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466651724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466651724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466651724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:11,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:11,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/aba337f956a1474a9c9c838946490352 2024-11-24T16:43:11,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/598bcb2761a742b0bfe5aae9ca25b8a4 is 50, key is test_row_0/C:col10/1732466590565/Put/seqid=0 2024-11-24T16:43:11,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742400_1576 (size=12001) 2024-11-24T16:43:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-24T16:43:12,204 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:12,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:43:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-24T16:43:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T16:43:12,206 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:43:12,207 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:43:12,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:43:12,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/598bcb2761a742b0bfe5aae9ca25b8a4 2024-11-24T16:43:12,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/5b9e2ae4e9294376b1b364ffc57157be as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5b9e2ae4e9294376b1b364ffc57157be 2024-11-24T16:43:12,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5b9e2ae4e9294376b1b364ffc57157be, entries=200, sequenceid=99, filesize=38.6 K 2024-11-24T16:43:12,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/aba337f956a1474a9c9c838946490352 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/aba337f956a1474a9c9c838946490352 2024-11-24T16:43:12,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/aba337f956a1474a9c9c838946490352, entries=150, sequenceid=99, filesize=11.7 K 2024-11-24T16:43:12,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/598bcb2761a742b0bfe5aae9ca25b8a4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/598bcb2761a742b0bfe5aae9ca25b8a4 2024-11-24T16:43:12,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/598bcb2761a742b0bfe5aae9ca25b8a4, entries=150, sequenceid=99, filesize=11.7 K 2024-11-24T16:43:12,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 8e3281f5174968dfe4494f3e286572d3 in 1662ms, sequenceid=99, compaction requested=true 2024-11-24T16:43:12,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:12,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:43:12,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:12,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:43:12,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:12,229 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:12,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:43:12,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:12,229 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:12,230 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:12,230 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:12,230 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/B is initiating minor compaction (all files) 2024-11-24T16:43:12,230 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/A is initiating minor compaction (all files) 2024-11-24T16:43:12,230 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/B in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:12,230 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/A in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:12,230 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/db926aa5939b4f668c5eba4a69c15867, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a9bc7afff8c549c6ad897c21f2ef94cb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/540d037bc9fe43e3accddd55235aae6b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5b9e2ae4e9294376b1b364ffc57157be] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=129.3 K 2024-11-24T16:43:12,230 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1eb1f0ae1ac6456383be5a08580a704e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e1cf10ee84234b96bd755d75d74f6589, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4e39aa5230134c4db03ec076d25fbbcd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/aba337f956a1474a9c9c838946490352] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=44.6 K 2024-11-24T16:43:12,230 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:12,230 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/db926aa5939b4f668c5eba4a69c15867, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a9bc7afff8c549c6ad897c21f2ef94cb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/540d037bc9fe43e3accddd55235aae6b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5b9e2ae4e9294376b1b364ffc57157be] 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb1f0ae1ac6456383be5a08580a704e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732466588038 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting db926aa5939b4f668c5eba4a69c15867, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732466588038 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e1cf10ee84234b96bd755d75d74f6589, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732466588134 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9bc7afff8c549c6ad897c21f2ef94cb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732466588134 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e39aa5230134c4db03ec076d25fbbcd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732466588772 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540d037bc9fe43e3accddd55235aae6b, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732466588772 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting aba337f956a1474a9c9c838946490352, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732466589398 2024-11-24T16:43:12,231 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b9e2ae4e9294376b1b364ffc57157be, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732466589385 2024-11-24T16:43:12,236 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:12,238 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:12,239 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/fb53624bdfa94eea94c02a6b69a7dba0 is 50, key is test_row_0/B:col10/1732466590565/Put/seqid=0 2024-11-24T16:43:12,242 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124c06fa79f6d9a45cb8a26386086418064_8e3281f5174968dfe4494f3e286572d3 store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:12,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742401_1577 (size=12139) 2024-11-24T16:43:12,244 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124c06fa79f6d9a45cb8a26386086418064_8e3281f5174968dfe4494f3e286572d3, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:12,244 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c06fa79f6d9a45cb8a26386086418064_8e3281f5174968dfe4494f3e286572d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:12,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742402_1578 (size=4469) 2024-11-24T16:43:12,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T16:43:12,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T16:43:12,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:12,359 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:43:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:12,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240578de832bb44106aeba5247cb52d48f_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466590568/Put/seqid=0 2024-11-24T16:43:12,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742403_1579 (size=12154) 2024-11-24T16:43:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T16:43:12,648 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/fb53624bdfa94eea94c02a6b69a7dba0 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fb53624bdfa94eea94c02a6b69a7dba0 2024-11-24T16:43:12,652 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/B of 8e3281f5174968dfe4494f3e286572d3 into fb53624bdfa94eea94c02a6b69a7dba0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:12,652 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:12,652 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/B, priority=12, startTime=1732466592229; duration=0sec 2024-11-24T16:43:12,652 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:12,652 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:B 2024-11-24T16:43:12,652 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:12,653 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:12,654 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/C is initiating minor compaction (all files) 2024-11-24T16:43:12,654 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/C in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:12,654 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/608f426f1802480b9fe0fc30f35062b6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/4dd8f85b6b7f4661beff8e15c0035e26, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/56919711ece140a99494481263a114ed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/598bcb2761a742b0bfe5aae9ca25b8a4] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=44.6 K 2024-11-24T16:43:12,654 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 608f426f1802480b9fe0fc30f35062b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732466588038 2024-11-24T16:43:12,655 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd8f85b6b7f4661beff8e15c0035e26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732466588134 2024-11-24T16:43:12,655 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 56919711ece140a99494481263a114ed, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732466588772 2024-11-24T16:43:12,655 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 598bcb2761a742b0bfe5aae9ca25b8a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732466589398 2024-11-24T16:43:12,662 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:12,663 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/e35f4574058e41c6a8e4f192e97e4e2e is 50, key is test_row_0/C:col10/1732466590565/Put/seqid=0 2024-11-24T16:43:12,663 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#A#compaction#491 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:12,664 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/4668e71f205543be8ddd6332c2e6b132 is 175, key is test_row_0/A:col10/1732466590565/Put/seqid=0 2024-11-24T16:43:12,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742404_1580 (size=12139) 2024-11-24T16:43:12,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742405_1581 (size=31093) 2024-11-24T16:43:12,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:12,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:12,772 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240578de832bb44106aeba5247cb52d48f_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240578de832bb44106aeba5247cb52d48f_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:12,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b49f362a336245bb81dd97cb3df44a52, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:12,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b49f362a336245bb81dd97cb3df44a52 is 175, key is test_row_0/A:col10/1732466590568/Put/seqid=0 2024-11-24T16:43:12,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742406_1582 (size=30955) 2024-11-24T16:43:12,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466592756, max timestamp=1732466592800Num uniques 859; 2024-11-24T16:43:12,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:12,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466592756, max timestamp=1732466592800Num uniques 859; 2024-11-24T16:43:12,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:12,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466592756, max timestamp=1732466592800Num uniques 859; 2024-11-24T16:43:12,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T16:43:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466652855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466652855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466652855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466652857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:12,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466652974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466652974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:12,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466652974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:12,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:12,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466652974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,072 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/4668e71f205543be8ddd6332c2e6b132 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/4668e71f205543be8ddd6332c2e6b132 2024-11-24T16:43:13,072 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/e35f4574058e41c6a8e4f192e97e4e2e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e35f4574058e41c6a8e4f192e97e4e2e 2024-11-24T16:43:13,076 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/A of 8e3281f5174968dfe4494f3e286572d3 into 4668e71f205543be8ddd6332c2e6b132(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:13,076 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/C of 8e3281f5174968dfe4494f3e286572d3 into e35f4574058e41c6a8e4f192e97e4e2e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:13,076 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:13,076 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:13,076 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/C, priority=12, startTime=1732466592229; duration=0sec 2024-11-24T16:43:13,076 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/A, priority=12, startTime=1732466592229; duration=0sec 2024-11-24T16:43:13,076 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:13,076 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:C 2024-11-24T16:43:13,076 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:13,076 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:A 2024-11-24T16:43:13,179 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b49f362a336245bb81dd97cb3df44a52 2024-11-24T16:43:13,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466653184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/e37a7b531f3b46b688495c484867fd1a is 50, key is test_row_0/B:col10/1732466590568/Put/seqid=0 2024-11-24T16:43:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466653184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742407_1583 (size=12001) 2024-11-24T16:43:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466653184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466653184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T16:43:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466653493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:13,589 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/e37a7b531f3b46b688495c484867fd1a 2024-11-24T16:43:13,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/72bf1ab85e474ca097292597e4df1243 is 50, key is test_row_0/C:col10/1732466590568/Put/seqid=0 2024-11-24T16:43:13,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742408_1584 (size=12001) 2024-11-24T16:43:13,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466653614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:13,618 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:13,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:14,000 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/72bf1ab85e474ca097292597e4df1243 2024-11-24T16:43:14,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b49f362a336245bb81dd97cb3df44a52 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b49f362a336245bb81dd97cb3df44a52 2024-11-24T16:43:14,007 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b49f362a336245bb81dd97cb3df44a52, entries=150, sequenceid=109, filesize=30.2 K 2024-11-24T16:43:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466654004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:14,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466654004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:14,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/e37a7b531f3b46b688495c484867fd1a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e37a7b531f3b46b688495c484867fd1a 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466654004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:14,011 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e37a7b531f3b46b688495c484867fd1a, entries=150, sequenceid=109, filesize=11.7 K 2024-11-24T16:43:14,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/72bf1ab85e474ca097292597e4df1243 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/72bf1ab85e474ca097292597e4df1243 2024-11-24T16:43:14,014 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/72bf1ab85e474ca097292597e4df1243, entries=150, sequenceid=109, filesize=11.7 K 2024-11-24T16:43:14,015 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8e3281f5174968dfe4494f3e286572d3 in 1656ms, sequenceid=109, compaction requested=false 2024-11-24T16:43:14,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:14,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-24T16:43:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-24T16:43:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:14,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-24T16:43:14,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-11-24T16:43:14,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8140 sec 2024-11-24T16:43:14,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466592825, max timestamp=1732466594016Num uniques 1018; 2024-11-24T16:43:14,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:43:14,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466592825, max timestamp=1732466594016Num uniques 1018; 2024-11-24T16:43:14,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:43:14,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466592825, max timestamp=1732466594016Num uniques 1018; 2024-11-24T16:43:14,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:43:14,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-24T16:43:14,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:14,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:14,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:14,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:14,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:14,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:14,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241de4d6e048da4a058d72d8204eb46c42_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466592834/Put/seqid=0 2024-11-24T16:43:14,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742409_1585 (size=14794) 2024-11-24T16:43:14,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:14,035 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241de4d6e048da4a058d72d8204eb46c42_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241de4d6e048da4a058d72d8204eb46c42_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:14,036 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/22d5e037f0344683a16a52011ac82182, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:14,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/22d5e037f0344683a16a52011ac82182 is 175, key is test_row_0/A:col10/1732466592834/Put/seqid=0 2024-11-24T16:43:14,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742410_1586 (size=39749) 2024-11-24T16:43:14,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/22d5e037f0344683a16a52011ac82182 2024-11-24T16:43:14,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/8eae09d69e824d02bfd7126f8c420798 is 50, key is test_row_0/B:col10/1732466592834/Put/seqid=0 2024-11-24T16:43:14,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742411_1587 (size=12151) 2024-11-24T16:43:14,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/8eae09d69e824d02bfd7126f8c420798 2024-11-24T16:43:14,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/765f0a9482994f40956b040d838e4a2b is 50, key is test_row_0/C:col10/1732466592834/Put/seqid=0 2024-11-24T16:43:14,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742412_1588 (size=12151) 2024-11-24T16:43:14,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466594051, max timestamp=1732466594099Num uniques 859; 2024-11-24T16:43:14,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466594051, max timestamp=1732466594099Num uniques 859; 2024-11-24T16:43:14,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:14,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:14,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466594051, max timestamp=1732466594099Num uniques 859; 2024-11-24T16:43:14,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:14,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466594139, max timestamp=1732466594189Num uniques 1018; 2024-11-24T16:43:14,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:43:14,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466594139, max timestamp=1732466594189Num uniques 1018; 2024-11-24T16:43:14,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732466594139, max timestamp=1732466594189Num uniques 1018; 2024-11-24T16:43:14,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:43:14,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T16:43:14,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732466594245, max timestamp=1732466594275Num uniques 1027; 2024-11-24T16:43:14,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-24T16:43:14,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732466594245, max timestamp=1732466594275Num uniques 1027; 2024-11-24T16:43:14,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732466594245, max timestamp=1732466594275Num uniques 1027; 2024-11-24T16:43:14,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-24T16:43:14,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-24T16:43:14,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T16:43:14,354 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:14,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:43:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-24T16:43:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T16:43:14,356 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:43:14,357 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:43:14,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:43:14,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732466594312, max timestamp=1732466594367Num uniques 1036; 2024-11-24T16:43:14,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-24T16:43:14,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732466594312, max timestamp=1732466594367Num uniques 1036; 2024-11-24T16:43:14,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-24T16:43:14,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732466594312, max timestamp=1732466594367Num uniques 1036; 2024-11-24T16:43:14,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-24T16:43:14,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732466594411, max timestamp=1732466594449Num uniques 1045; 2024-11-24T16:43:14,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-24T16:43:14,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732466594411, max timestamp=1732466594449Num uniques 1045; 2024-11-24T16:43:14,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-24T16:43:14,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732466594411, max timestamp=1732466594449Num uniques 1045; 2024-11-24T16:43:14,454 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-24T16:43:14,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/765f0a9482994f40956b040d838e4a2b 2024-11-24T16:43:14,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/22d5e037f0344683a16a52011ac82182 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/22d5e037f0344683a16a52011ac82182 2024-11-24T16:43:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T16:43:14,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/22d5e037f0344683a16a52011ac82182, entries=200, sequenceid=150, filesize=38.8 K 2024-11-24T16:43:14,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/8eae09d69e824d02bfd7126f8c420798 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/8eae09d69e824d02bfd7126f8c420798 2024-11-24T16:43:14,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/8eae09d69e824d02bfd7126f8c420798, entries=150, sequenceid=150, filesize=11.9 K 2024-11-24T16:43:14,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/765f0a9482994f40956b040d838e4a2b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/765f0a9482994f40956b040d838e4a2b 2024-11-24T16:43:14,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/765f0a9482994f40956b040d838e4a2b, entries=150, sequenceid=150, filesize=11.9 K 2024-11-24T16:43:14,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=60.38 KB/61830 for 8e3281f5174968dfe4494f3e286572d3 in 454ms, sequenceid=150, compaction requested=true 2024-11-24T16:43:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:43:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:43:14,476 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:43:14,476 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:14,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/B is initiating minor compaction (all files) 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/A is initiating minor compaction (all files) 2024-11-24T16:43:14,477 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/B in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,477 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/A in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,477 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fb53624bdfa94eea94c02a6b69a7dba0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e37a7b531f3b46b688495c484867fd1a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/8eae09d69e824d02bfd7126f8c420798] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=35.4 K 2024-11-24T16:43:14,477 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/4668e71f205543be8ddd6332c2e6b132, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b49f362a336245bb81dd97cb3df44a52, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/22d5e037f0344683a16a52011ac82182] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=99.4 K 2024-11-24T16:43:14,477 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/4668e71f205543be8ddd6332c2e6b132, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b49f362a336245bb81dd97cb3df44a52, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/22d5e037f0344683a16a52011ac82182] 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fb53624bdfa94eea94c02a6b69a7dba0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732466589398 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4668e71f205543be8ddd6332c2e6b132, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732466589398 2024-11-24T16:43:14,477 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e37a7b531f3b46b688495c484867fd1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732466590568 2024-11-24T16:43:14,478 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b49f362a336245bb81dd97cb3df44a52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732466590568 2024-11-24T16:43:14,478 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eae09d69e824d02bfd7126f8c420798, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732466592825 2024-11-24T16:43:14,478 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d5e037f0344683a16a52011ac82182, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732466592825 2024-11-24T16:43:14,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:14,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=67.09 KB heapSize=182.39 KB 2024-11-24T16:43:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:14,483 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:14,484 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#B#compaction#500 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:14,484 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/50893fecde7a4269b9ce6339611bcdd8 is 50, key is test_row_0/B:col10/1732466592834/Put/seqid=0 2024-11-24T16:43:14,485 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124c60677430ed84e5dbd2311c5486e16b5_8e3281f5174968dfe4494f3e286572d3 store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:14,487 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124c60677430ed84e5dbd2311c5486e16b5_8e3281f5174968dfe4494f3e286572d3, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:14,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124927ab049134c45b6a81c5282a3a22edb_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466594438/Put/seqid=0 2024-11-24T16:43:14,487 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c60677430ed84e5dbd2311c5486e16b5_8e3281f5174968dfe4494f3e286572d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:14,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742413_1589 (size=12391) 2024-11-24T16:43:14,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742415_1591 (size=4469) 2024-11-24T16:43:14,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742414_1590 (size=12304) 2024-11-24T16:43:14,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:14,500 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/50893fecde7a4269b9ce6339611bcdd8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/50893fecde7a4269b9ce6339611bcdd8 2024-11-24T16:43:14,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124927ab049134c45b6a81c5282a3a22edb_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124927ab049134c45b6a81c5282a3a22edb_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:14,502 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/16fd28dc382d436fbdef692853db7f8d, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:14,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/16fd28dc382d436fbdef692853db7f8d is 175, key is test_row_0/A:col10/1732466594438/Put/seqid=0 2024-11-24T16:43:14,504 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/B of 8e3281f5174968dfe4494f3e286572d3 into 50893fecde7a4269b9ce6339611bcdd8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:14,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:14,504 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/B, priority=13, startTime=1732466594476; duration=0sec 2024-11-24T16:43:14,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:14,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:B 2024-11-24T16:43:14,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:14,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:14,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/C is initiating minor compaction (all files) 2024-11-24T16:43:14,505 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/C in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,505 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e35f4574058e41c6a8e4f192e97e4e2e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/72bf1ab85e474ca097292597e4df1243, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/765f0a9482994f40956b040d838e4a2b] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=35.4 K 2024-11-24T16:43:14,506 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting e35f4574058e41c6a8e4f192e97e4e2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732466589398 2024-11-24T16:43:14,506 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 72bf1ab85e474ca097292597e4df1243, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732466590568 2024-11-24T16:43:14,506 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 765f0a9482994f40956b040d838e4a2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732466592825 2024-11-24T16:43:14,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T16:43:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,515 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:14,515 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/db1c0296c81448a7914177680e3047e4 is 50, key is test_row_0/C:col10/1732466592834/Put/seqid=0 2024-11-24T16:43:14,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742416_1592 (size=31105) 2024-11-24T16:43:14,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742417_1593 (size=12391) 2024-11-24T16:43:14,533 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/db1c0296c81448a7914177680e3047e4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/db1c0296c81448a7914177680e3047e4 2024-11-24T16:43:14,537 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/C of 8e3281f5174968dfe4494f3e286572d3 into db1c0296c81448a7914177680e3047e4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:14,537 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:14,537 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/C, priority=13, startTime=1732466594476; duration=0sec 2024-11-24T16:43:14,537 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:14,537 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:C 2024-11-24T16:43:14,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466654583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:14,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T16:43:14,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:14,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T16:43:14,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466654693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:14,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T16:43:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:14,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,894 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:14,895 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a850da2d5d904e55b6f2367e545e659b is 175, key is test_row_0/A:col10/1732466592834/Put/seqid=0 2024-11-24T16:43:14,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742418_1594 (size=31345) 2024-11-24T16:43:14,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466654904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:14,928 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/16fd28dc382d436fbdef692853db7f8d 2024-11-24T16:43:14,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/fdfd1e52c44649a980da90f1a68e4c00 is 50, key is test_row_0/B:col10/1732466594438/Put/seqid=0 2024-11-24T16:43:14,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742419_1595 (size=12151) 2024-11-24T16:43:14,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/fdfd1e52c44649a980da90f1a68e4c00 2024-11-24T16:43:14,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/5fcc90b9c51941da8a05d2d5669cf2aa is 50, key is test_row_0/C:col10/1732466594438/Put/seqid=0 2024-11-24T16:43:14,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742420_1596 (size=12151) 2024-11-24T16:43:14,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T16:43:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:14,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T16:43:15,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:15,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466655024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466655024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:15,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466655024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:15,029 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:15,029 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:15,029 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:15,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T16:43:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:15,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:15,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466655213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:15,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T16:43:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:15,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:15,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:15,302 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a850da2d5d904e55b6f2367e545e659b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a850da2d5d904e55b6f2367e545e659b 2024-11-24T16:43:15,306 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/A of 8e3281f5174968dfe4494f3e286572d3 into a850da2d5d904e55b6f2367e545e659b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:15,306 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:15,306 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/A, priority=13, startTime=1732466594476; duration=0sec 2024-11-24T16:43:15,306 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:15,306 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:A 2024-11-24T16:43:15,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/5fcc90b9c51941da8a05d2d5669cf2aa 2024-11-24T16:43:15,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/16fd28dc382d436fbdef692853db7f8d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/16fd28dc382d436fbdef692853db7f8d 2024-11-24T16:43:15,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/16fd28dc382d436fbdef692853db7f8d, entries=150, sequenceid=245, filesize=30.4 K 2024-11-24T16:43:15,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/fdfd1e52c44649a980da90f1a68e4c00 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fdfd1e52c44649a980da90f1a68e4c00 2024-11-24T16:43:15,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fdfd1e52c44649a980da90f1a68e4c00, entries=150, sequenceid=245, filesize=11.9 K 2024-11-24T16:43:15,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/5fcc90b9c51941da8a05d2d5669cf2aa as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/5fcc90b9c51941da8a05d2d5669cf2aa 2024-11-24T16:43:15,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/5fcc90b9c51941da8a05d2d5669cf2aa, entries=150, sequenceid=245, filesize=11.9 K 2024-11-24T16:43:15,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~182.34 KB/186720, currentSize=114.05 KB/116790 for 8e3281f5174968dfe4494f3e286572d3 in 888ms, sequenceid=245, compaction requested=false 2024-11-24T16:43:15,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:15,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T16:43:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:15,427 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T16:43:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:15,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241f66d5215a1c4c22bd8558f1944d9421_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466594581/Put/seqid=0 2024-11-24T16:43:15,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742421_1597 (size=12454) 2024-11-24T16:43:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T16:43:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:15,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:15,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466655763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:15,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:15,841 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241f66d5215a1c4c22bd8558f1944d9421_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241f66d5215a1c4c22bd8558f1944d9421_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:15,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/cc1c87e8ebe545e78bb9ff15260515c1, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:15,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/cc1c87e8ebe545e78bb9ff15260515c1 is 175, key is test_row_0/A:col10/1732466594581/Put/seqid=0 2024-11-24T16:43:15,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742422_1598 (size=31255) 2024-11-24T16:43:15,847 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/cc1c87e8ebe545e78bb9ff15260515c1 2024-11-24T16:43:15,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/de1ec78bc55f470d94c31a0f5c89bbdc is 50, key is test_row_0/B:col10/1732466594581/Put/seqid=0 2024-11-24T16:43:15,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742423_1599 (size=12301) 2024-11-24T16:43:15,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/de1ec78bc55f470d94c31a0f5c89bbdc 2024-11-24T16:43:15,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/15125faa8f8b4defa41e39ddad2e3a44 is 50, key is test_row_0/C:col10/1732466594581/Put/seqid=0 2024-11-24T16:43:15,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742424_1600 (size=12301) 2024-11-24T16:43:15,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:15,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466655873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466656084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:16,269 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/15125faa8f8b4defa41e39ddad2e3a44 2024-11-24T16:43:16,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/cc1c87e8ebe545e78bb9ff15260515c1 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/cc1c87e8ebe545e78bb9ff15260515c1 2024-11-24T16:43:16,276 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/cc1c87e8ebe545e78bb9ff15260515c1, entries=150, sequenceid=268, filesize=30.5 K 2024-11-24T16:43:16,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/de1ec78bc55f470d94c31a0f5c89bbdc as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/de1ec78bc55f470d94c31a0f5c89bbdc 2024-11-24T16:43:16,280 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/de1ec78bc55f470d94c31a0f5c89bbdc, entries=150, sequenceid=268, filesize=12.0 K 2024-11-24T16:43:16,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/15125faa8f8b4defa41e39ddad2e3a44 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/15125faa8f8b4defa41e39ddad2e3a44 2024-11-24T16:43:16,283 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/15125faa8f8b4defa41e39ddad2e3a44, entries=150, sequenceid=268, filesize=12.0 K 2024-11-24T16:43:16,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8e3281f5174968dfe4494f3e286572d3 in 857ms, sequenceid=268, compaction requested=true 2024-11-24T16:43:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-24T16:43:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-24T16:43:16,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-24T16:43:16,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9280 sec 2024-11-24T16:43:16,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9330 sec 2024-11-24T16:43:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:16,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:43:16,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:16,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:16,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:16,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:16,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:16,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:16,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fdf5c4b99cc140449c8ba6f2b045faf5_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466595725/Put/seqid=0 2024-11-24T16:43:16,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742425_1601 (size=14994) 2024-11-24T16:43:16,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466656469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T16:43:16,514 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:16,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:43:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-24T16:43:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T16:43:16,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:43:16,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:43:16,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:43:16,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466656584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T16:43:16,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:16,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:16,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:16,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:16,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466656794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:16,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:16,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:16,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fdf5c4b99cc140449c8ba6f2b045faf5_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fdf5c4b99cc140449c8ba6f2b045faf5_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:16,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,822 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/0c50188748144be1be816648a5bb769d, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/0c50188748144be1be816648a5bb769d is 175, key is test_row_0/A:col10/1732466595725/Put/seqid=0 2024-11-24T16:43:16,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742426_1602 (size=39949) 2024-11-24T16:43:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T16:43:16,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:16,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466657044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:17,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466657044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:17,046 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:17,046 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:17,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466657054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:17,057 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:17,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466657103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:17,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:17,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T16:43:17,228 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/0c50188748144be1be816648a5bb769d 2024-11-24T16:43:17,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5e6a9e548e084fa9baaee606f0830997 is 50, key is test_row_0/B:col10/1732466595725/Put/seqid=0 2024-11-24T16:43:17,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742427_1603 (size=12301) 2024-11-24T16:43:17,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:17,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466657614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:17,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5e6a9e548e084fa9baaee606f0830997 2024-11-24T16:43:17,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/f11af04c71084358ac4155f52a1480af is 50, key is test_row_0/C:col10/1732466595725/Put/seqid=0 2024-11-24T16:43:17,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466657643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:17,645 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:17,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742428_1604 (size=12301) 2024-11-24T16:43:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T16:43:17,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:17,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:17,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:18,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:18,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:18,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:18,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:18,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:18,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:18,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/f11af04c71084358ac4155f52a1480af 2024-11-24T16:43:18,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/0c50188748144be1be816648a5bb769d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/0c50188748144be1be816648a5bb769d 2024-11-24T16:43:18,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/0c50188748144be1be816648a5bb769d, entries=200, sequenceid=281, filesize=39.0 K 2024-11-24T16:43:18,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5e6a9e548e084fa9baaee606f0830997 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5e6a9e548e084fa9baaee606f0830997 2024-11-24T16:43:18,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5e6a9e548e084fa9baaee606f0830997, entries=150, sequenceid=281, filesize=12.0 K 2024-11-24T16:43:18,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/f11af04c71084358ac4155f52a1480af as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/f11af04c71084358ac4155f52a1480af 2024-11-24T16:43:18,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/f11af04c71084358ac4155f52a1480af, entries=150, sequenceid=281, filesize=12.0 K 2024-11-24T16:43:18,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8e3281f5174968dfe4494f3e286572d3 in 1671ms, sequenceid=281, compaction requested=true 2024-11-24T16:43:18,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:18,068 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:18,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:43:18,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:18,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:43:18,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:18,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:43:18,068 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:18,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:18,069 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:18,069 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/A is initiating minor compaction (all files) 2024-11-24T16:43:18,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:18,069 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/A in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:18,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/B is initiating minor compaction (all files) 2024-11-24T16:43:18,069 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/B in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:18,069 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a850da2d5d904e55b6f2367e545e659b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/16fd28dc382d436fbdef692853db7f8d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/cc1c87e8ebe545e78bb9ff15260515c1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/0c50188748144be1be816648a5bb769d] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=130.5 K 2024-11-24T16:43:18,069 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/50893fecde7a4269b9ce6339611bcdd8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fdfd1e52c44649a980da90f1a68e4c00, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/de1ec78bc55f470d94c31a0f5c89bbdc, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5e6a9e548e084fa9baaee606f0830997] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=48.0 K 2024-11-24T16:43:18,069 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:18,069 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a850da2d5d904e55b6f2367e545e659b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/16fd28dc382d436fbdef692853db7f8d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/cc1c87e8ebe545e78bb9ff15260515c1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/0c50188748144be1be816648a5bb769d] 2024-11-24T16:43:18,069 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 50893fecde7a4269b9ce6339611bcdd8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732466592825 2024-11-24T16:43:18,070 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a850da2d5d904e55b6f2367e545e659b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732466592825 2024-11-24T16:43:18,070 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting fdfd1e52c44649a980da90f1a68e4c00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732466594438 2024-11-24T16:43:18,070 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16fd28dc382d436fbdef692853db7f8d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732466594438 2024-11-24T16:43:18,070 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting de1ec78bc55f470d94c31a0f5c89bbdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732466594536 2024-11-24T16:43:18,070 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e6a9e548e084fa9baaee606f0830997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732466595725 2024-11-24T16:43:18,070 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1c87e8ebe545e78bb9ff15260515c1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732466594536 2024-11-24T16:43:18,071 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c50188748144be1be816648a5bb769d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732466595725 2024-11-24T16:43:18,078 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#B#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:18,079 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/bcbe4f04271748d2a62d616f80822c29 is 50, key is test_row_0/B:col10/1732466595725/Put/seqid=0 2024-11-24T16:43:18,080 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:18,082 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124dc7ff2143fd741ecaa28ebb5727cb308_8e3281f5174968dfe4494f3e286572d3 store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:18,085 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124dc7ff2143fd741ecaa28ebb5727cb308_8e3281f5174968dfe4494f3e286572d3, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:18,086 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124dc7ff2143fd741ecaa28ebb5727cb308_8e3281f5174968dfe4494f3e286572d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:18,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742429_1605 (size=12677) 2024-11-24T16:43:18,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742430_1606 (size=4469) 2024-11-24T16:43:18,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T16:43:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:18,199 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T16:43:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124626c08be5ee143638c5f5a66f791adf5_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466596467/Put/seqid=0 2024-11-24T16:43:18,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742431_1607 (size=12454) 2024-11-24T16:43:18,496 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#A#compaction#513 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:18,497 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/5e9a2f1071d94827a3c578f316bc2c7b is 175, key is test_row_0/A:col10/1732466595725/Put/seqid=0 2024-11-24T16:43:18,498 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/bcbe4f04271748d2a62d616f80822c29 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/bcbe4f04271748d2a62d616f80822c29 2024-11-24T16:43:18,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742432_1608 (size=31631) 2024-11-24T16:43:18,503 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/B of 8e3281f5174968dfe4494f3e286572d3 into bcbe4f04271748d2a62d616f80822c29(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:18,503 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:18,503 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/B, priority=12, startTime=1732466598068; duration=0sec 2024-11-24T16:43:18,503 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:18,503 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:B 2024-11-24T16:43:18,503 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:18,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:18,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/C is initiating minor compaction (all files) 2024-11-24T16:43:18,504 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/C in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:18,504 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/db1c0296c81448a7914177680e3047e4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/5fcc90b9c51941da8a05d2d5669cf2aa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/15125faa8f8b4defa41e39ddad2e3a44, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/f11af04c71084358ac4155f52a1480af] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=48.0 K 2024-11-24T16:43:18,504 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting db1c0296c81448a7914177680e3047e4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732466592825 2024-11-24T16:43:18,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fcc90b9c51941da8a05d2d5669cf2aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732466594438 2024-11-24T16:43:18,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 15125faa8f8b4defa41e39ddad2e3a44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732466594536 2024-11-24T16:43:18,505 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting f11af04c71084358ac4155f52a1480af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732466595725 2024-11-24T16:43:18,512 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#C#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:18,512 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/1551462958ec4286ba6ccf6ed5076f7a is 50, key is test_row_0/C:col10/1732466595725/Put/seqid=0 2024-11-24T16:43:18,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742433_1609 (size=12677) 2024-11-24T16:43:18,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:18,615 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124626c08be5ee143638c5f5a66f791adf5_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124626c08be5ee143638c5f5a66f791adf5_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:18,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2a5b8a60fce34f5082503447705eb4ba, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:18,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2a5b8a60fce34f5082503447705eb4ba is 175, key is test_row_0/A:col10/1732466596467/Put/seqid=0 2024-11-24T16:43:18,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742434_1610 (size=31255) 2024-11-24T16:43:18,621 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2a5b8a60fce34f5082503447705eb4ba 2024-11-24T16:43:18,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/9b831dc3c279446c9bc2e6c8801c0461 is 50, key is test_row_0/B:col10/1732466596467/Put/seqid=0 2024-11-24T16:43:18,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742435_1611 (size=12301) 2024-11-24T16:43:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:18,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T16:43:18,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466658681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:18,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466658794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:18,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:18,904 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/5e9a2f1071d94827a3c578f316bc2c7b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5e9a2f1071d94827a3c578f316bc2c7b 2024-11-24T16:43:18,908 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/A of 8e3281f5174968dfe4494f3e286572d3 into 5e9a2f1071d94827a3c578f316bc2c7b(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:18,908 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:18,908 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/A, priority=12, startTime=1732466598068; duration=0sec 2024-11-24T16:43:18,908 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:18,908 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:A 2024-11-24T16:43:18,919 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/1551462958ec4286ba6ccf6ed5076f7a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/1551462958ec4286ba6ccf6ed5076f7a 2024-11-24T16:43:18,922 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/C of 8e3281f5174968dfe4494f3e286572d3 into 1551462958ec4286ba6ccf6ed5076f7a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:18,922 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:18,922 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/C, priority=12, startTime=1732466598068; duration=0sec 2024-11-24T16:43:18,922 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:18,922 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:C 2024-11-24T16:43:19,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466659003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:19,030 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/9b831dc3c279446c9bc2e6c8801c0461 2024-11-24T16:43:19,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/ef1d653358f547eab1ba17082a586784 is 50, key is test_row_0/C:col10/1732466596467/Put/seqid=0 2024-11-24T16:43:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742436_1612 (size=12301) 2024-11-24T16:43:19,041 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/ef1d653358f547eab1ba17082a586784 2024-11-24T16:43:19,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2a5b8a60fce34f5082503447705eb4ba as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2a5b8a60fce34f5082503447705eb4ba 2024-11-24T16:43:19,048 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2a5b8a60fce34f5082503447705eb4ba, entries=150, sequenceid=300, filesize=30.5 K 2024-11-24T16:43:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/9b831dc3c279446c9bc2e6c8801c0461 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/9b831dc3c279446c9bc2e6c8801c0461 2024-11-24T16:43:19,051 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/9b831dc3c279446c9bc2e6c8801c0461, entries=150, sequenceid=300, filesize=12.0 K 2024-11-24T16:43:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/ef1d653358f547eab1ba17082a586784 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/ef1d653358f547eab1ba17082a586784 2024-11-24T16:43:19,059 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/ef1d653358f547eab1ba17082a586784, entries=150, sequenceid=300, filesize=12.0 K 2024-11-24T16:43:19,060 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8e3281f5174968dfe4494f3e286572d3 in 861ms, sequenceid=300, compaction requested=false 2024-11-24T16:43:19,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:19,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:19,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-24T16:43:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-24T16:43:19,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-24T16:43:19,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5450 sec 2024-11-24T16:43:19,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.5490 sec 2024-11-24T16:43:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:19,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T16:43:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:19,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cd18f2d15845457f913b0b0661ba1380_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466598674/Put/seqid=0 2024-11-24T16:43:19,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742437_1613 (size=12454) 2024-11-24T16:43:19,328 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:19,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cd18f2d15845457f913b0b0661ba1380_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd18f2d15845457f913b0b0661ba1380_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:19,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/655b101ca6fb47a4b216737136a52f24, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:19,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/655b101ca6fb47a4b216737136a52f24 is 175, key is test_row_0/A:col10/1732466598674/Put/seqid=0 2024-11-24T16:43:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742438_1614 (size=31255) 2024-11-24T16:43:19,352 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/655b101ca6fb47a4b216737136a52f24 2024-11-24T16:43:19,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5cd0a1da6aa74a2fbd5e34a428cc333e is 50, key is test_row_0/B:col10/1732466598674/Put/seqid=0 2024-11-24T16:43:19,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742439_1615 (size=12301) 2024-11-24T16:43:19,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5cd0a1da6aa74a2fbd5e34a428cc333e 2024-11-24T16:43:19,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/8db8a313757e4285a9587eda8ad04caf is 50, key is test_row_0/C:col10/1732466598674/Put/seqid=0 2024-11-24T16:43:19,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742440_1616 (size=12301) 2024-11-24T16:43:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466659394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:19,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466659504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:19,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466659714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:19,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/8db8a313757e4285a9587eda8ad04caf 2024-11-24T16:43:19,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/655b101ca6fb47a4b216737136a52f24 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/655b101ca6fb47a4b216737136a52f24 2024-11-24T16:43:19,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/655b101ca6fb47a4b216737136a52f24, entries=150, sequenceid=317, filesize=30.5 K 2024-11-24T16:43:19,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5cd0a1da6aa74a2fbd5e34a428cc333e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5cd0a1da6aa74a2fbd5e34a428cc333e 2024-11-24T16:43:19,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5cd0a1da6aa74a2fbd5e34a428cc333e, entries=150, sequenceid=317, filesize=12.0 K 2024-11-24T16:43:19,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/8db8a313757e4285a9587eda8ad04caf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/8db8a313757e4285a9587eda8ad04caf 2024-11-24T16:43:19,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/8db8a313757e4285a9587eda8ad04caf, entries=150, sequenceid=317, filesize=12.0 K 2024-11-24T16:43:19,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8e3281f5174968dfe4494f3e286572d3 in 476ms, sequenceid=317, compaction requested=true 2024-11-24T16:43:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:43:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:43:19,792 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:43:19,792 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/A is initiating minor compaction (all files) 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/B is initiating minor compaction (all files) 2024-11-24T16:43:19,793 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/A in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:19,793 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/B in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:19,793 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5e9a2f1071d94827a3c578f316bc2c7b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2a5b8a60fce34f5082503447705eb4ba, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/655b101ca6fb47a4b216737136a52f24] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=91.9 K 2024-11-24T16:43:19,793 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:19,793 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/bcbe4f04271748d2a62d616f80822c29, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/9b831dc3c279446c9bc2e6c8801c0461, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5cd0a1da6aa74a2fbd5e34a428cc333e] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=36.4 K 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5e9a2f1071d94827a3c578f316bc2c7b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2a5b8a60fce34f5082503447705eb4ba, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/655b101ca6fb47a4b216737136a52f24] 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting bcbe4f04271748d2a62d616f80822c29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732466595725 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e9a2f1071d94827a3c578f316bc2c7b, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732466595725 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b831dc3c279446c9bc2e6c8801c0461, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732466596422 2024-11-24T16:43:19,793 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a5b8a60fce34f5082503447705eb4ba, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732466596422 2024-11-24T16:43:19,794 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 655b101ca6fb47a4b216737136a52f24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732466598664 2024-11-24T16:43:19,794 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd0a1da6aa74a2fbd5e34a428cc333e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732466598664 2024-11-24T16:43:19,798 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:19,799 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:19,799 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411244d8fc687a18847cbbb4d87e0e148d10d_8e3281f5174968dfe4494f3e286572d3 store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:19,800 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/610422a6d4a14cbf810d2afde9862983 is 50, key is test_row_0/B:col10/1732466598674/Put/seqid=0 2024-11-24T16:43:19,801 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411244d8fc687a18847cbbb4d87e0e148d10d_8e3281f5174968dfe4494f3e286572d3, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:19,801 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244d8fc687a18847cbbb4d87e0e148d10d_8e3281f5174968dfe4494f3e286572d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:19,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742442_1618 (size=4469) 2024-11-24T16:43:19,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742441_1617 (size=12779) 2024-11-24T16:43:19,824 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/610422a6d4a14cbf810d2afde9862983 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/610422a6d4a14cbf810d2afde9862983 2024-11-24T16:43:19,828 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/B of 8e3281f5174968dfe4494f3e286572d3 into 610422a6d4a14cbf810d2afde9862983(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:19,828 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:19,828 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/B, priority=13, startTime=1732466599792; duration=0sec 2024-11-24T16:43:19,828 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:19,828 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:B 2024-11-24T16:43:19,829 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:19,829 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:19,829 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/C is initiating minor compaction (all files) 2024-11-24T16:43:19,829 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/C in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:19,830 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/1551462958ec4286ba6ccf6ed5076f7a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/ef1d653358f547eab1ba17082a586784, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/8db8a313757e4285a9587eda8ad04caf] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=36.4 K 2024-11-24T16:43:19,830 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1551462958ec4286ba6ccf6ed5076f7a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732466595725 2024-11-24T16:43:19,830 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting ef1d653358f547eab1ba17082a586784, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732466596422 2024-11-24T16:43:19,831 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db8a313757e4285a9587eda8ad04caf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732466598664 2024-11-24T16:43:19,836 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#C#compaction#523 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:19,836 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/294b1b2a160b44f2bb3e5f1cace69f60 is 50, key is test_row_0/C:col10/1732466598674/Put/seqid=0 2024-11-24T16:43:19,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742443_1619 (size=12779) 2024-11-24T16:43:19,844 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/294b1b2a160b44f2bb3e5f1cace69f60 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/294b1b2a160b44f2bb3e5f1cace69f60 2024-11-24T16:43:19,849 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/C of 8e3281f5174968dfe4494f3e286572d3 into 294b1b2a160b44f2bb3e5f1cace69f60(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:19,849 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:19,849 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/C, priority=13, startTime=1732466599792; duration=0sec 2024-11-24T16:43:19,849 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:19,849 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:C 2024-11-24T16:43:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:20,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T16:43:20,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:20,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:20,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:20,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:20,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:20,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:20,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124904f3a4403164761be476e21cdf9eb24_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466600025/Put/seqid=0 2024-11-24T16:43:20,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742444_1620 (size=14994) 2024-11-24T16:43:20,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466660074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:20,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466660183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:20,221 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:20,221 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/ccb424a2fe344df6b70b2eb027e479fd is 175, key is test_row_0/A:col10/1732466598674/Put/seqid=0 2024-11-24T16:43:20,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742445_1621 (size=31733) 2024-11-24T16:43:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466660394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:20,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:20,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124904f3a4403164761be476e21cdf9eb24_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124904f3a4403164761be476e21cdf9eb24_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:20,442 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/90d0f5a7c36f42d498f761b38a8ca85c, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:20,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/90d0f5a7c36f42d498f761b38a8ca85c is 175, key is test_row_0/A:col10/1732466600025/Put/seqid=0 2024-11-24T16:43:20,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742446_1622 (size=39949) 2024-11-24T16:43:20,446 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/90d0f5a7c36f42d498f761b38a8ca85c 2024-11-24T16:43:20,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/1805147183554fd4bc58fe46dcf0c6e9 is 50, key is test_row_0/B:col10/1732466600025/Put/seqid=0 2024-11-24T16:43:20,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742447_1623 (size=12301) 2024-11-24T16:43:20,629 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/ccb424a2fe344df6b70b2eb027e479fd as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ccb424a2fe344df6b70b2eb027e479fd 2024-11-24T16:43:20,633 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/A of 8e3281f5174968dfe4494f3e286572d3 into ccb424a2fe344df6b70b2eb027e479fd(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:20,633 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:20,633 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/A, priority=13, startTime=1732466599792; duration=0sec 2024-11-24T16:43:20,633 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:20,633 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:A 2024-11-24T16:43:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T16:43:20,684 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:20,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-24T16:43:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T16:43:20,686 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:43:20,687 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:43:20,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:43:20,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:20,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466660704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T16:43:20,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T16:43:20,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:20,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:20,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:20,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:20,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:20,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/1805147183554fd4bc58fe46dcf0c6e9 2024-11-24T16:43:20,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f is 50, key is test_row_0/C:col10/1732466600025/Put/seqid=0 2024-11-24T16:43:20,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742448_1624 (size=12301) 2024-11-24T16:43:20,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T16:43:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:20,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:20,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:20,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T16:43:21,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:21,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39632 deadline: 1732466661084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39652 deadline: 1732466661084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:21,086 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:21,086 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39686 deadline: 1732466661094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:21,099 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_0' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:21,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T16:43:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:21,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:21,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:21,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466661214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:21,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:21,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f 2024-11-24T16:43:21,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/90d0f5a7c36f42d498f761b38a8ca85c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/90d0f5a7c36f42d498f761b38a8ca85c 2024-11-24T16:43:21,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/90d0f5a7c36f42d498f761b38a8ca85c, entries=200, sequenceid=338, filesize=39.0 K 2024-11-24T16:43:21,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/1805147183554fd4bc58fe46dcf0c6e9 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1805147183554fd4bc58fe46dcf0c6e9 2024-11-24T16:43:21,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1805147183554fd4bc58fe46dcf0c6e9, entries=150, sequenceid=338, filesize=12.0 K 2024-11-24T16:43:21,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f 2024-11-24T16:43:21,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f, entries=150, sequenceid=338, filesize=12.0 K 2024-11-24T16:43:21,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8e3281f5174968dfe4494f3e286572d3 in 1271ms, sequenceid=338, compaction requested=false 2024-11-24T16:43:21,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:21,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T16:43:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:21,299 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T16:43:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T16:43:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246a3ec342b76b43bd9f59e26c77a7ca97_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466600072/Put/seqid=0 2024-11-24T16:43:21,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742449_1625 (size=12454) 2024-11-24T16:43:21,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:21,373 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246a3ec342b76b43bd9f59e26c77a7ca97_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a3ec342b76b43bd9f59e26c77a7ca97_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2624fef9cf384f069b8a06ea535e0521, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2624fef9cf384f069b8a06ea535e0521 is 175, key is test_row_0/A:col10/1732466600072/Put/seqid=0 2024-11-24T16:43:21,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742450_1626 (size=31255) 2024-11-24T16:43:21,389 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2624fef9cf384f069b8a06ea535e0521 2024-11-24T16:43:21,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/686f2a43287b4448b7e33b2789d85cbf is 50, key is test_row_0/B:col10/1732466600072/Put/seqid=0 2024-11-24T16:43:21,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742451_1627 (size=12301) 2024-11-24T16:43:21,815 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/686f2a43287b4448b7e33b2789d85cbf 2024-11-24T16:43:21,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/0c33830ee90443bfbc0368653f53b54e is 50, key is test_row_0/C:col10/1732466600072/Put/seqid=0 2024-11-24T16:43:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T16:43:21,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742452_1628 (size=12301) 2024-11-24T16:43:22,226 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/0c33830ee90443bfbc0368653f53b54e 2024-11-24T16:43:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:22,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:22,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2624fef9cf384f069b8a06ea535e0521 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2624fef9cf384f069b8a06ea535e0521 2024-11-24T16:43:22,234 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2624fef9cf384f069b8a06ea535e0521, entries=150, sequenceid=352, filesize=30.5 K 2024-11-24T16:43:22,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/686f2a43287b4448b7e33b2789d85cbf as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/686f2a43287b4448b7e33b2789d85cbf 2024-11-24T16:43:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,238 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/686f2a43287b4448b7e33b2789d85cbf, entries=150, sequenceid=352, filesize=12.0 K 2024-11-24T16:43:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/0c33830ee90443bfbc0368653f53b54e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0c33830ee90443bfbc0368653f53b54e 2024-11-24T16:43:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,242 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0c33830ee90443bfbc0368653f53b54e, entries=150, sequenceid=352, filesize=12.0 K 2024-11-24T16:43:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,242 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for 8e3281f5174968dfe4494f3e286572d3 in 943ms, sequenceid=352, compaction requested=true 2024-11-24T16:43:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:22,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-24T16:43:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-24T16:43:22,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-24T16:43:22,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-24T16:43:22,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.5610 sec 2024-11-24T16:43:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:22,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:43:22,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:22,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:22,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:22,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:22,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:22,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:22,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124add438e10d5d40069fcefc688c462e6c_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466602247/Put/seqid=0 2024-11-24T16:43:22,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742453_1629 (size=14994) 2024-11-24T16:43:22,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466602291, max timestamp=1732466602344Num uniques 859; 2024-11-24T16:43:22,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:22,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466602291, max timestamp=1732466602344Num uniques 859; 2024-11-24T16:43:22,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:22,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466602291, max timestamp=1732466602344Num uniques 859; 2024-11-24T16:43:22,348 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466662450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:22,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:22,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466662563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:22,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:22,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:22,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:22,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124add438e10d5d40069fcefc688c462e6c_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124add438e10d5d40069fcefc688c462e6c_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:22,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a7c08a6b1abb4a248aa8fc992733ddd4, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:22,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a7c08a6b1abb4a248aa8fc992733ddd4 is 175, key is test_row_0/A:col10/1732466602247/Put/seqid=0 2024-11-24T16:43:22,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742454_1630 (size=39949) 2024-11-24T16:43:22,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466662774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T16:43:22,834 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:22,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:43:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-24T16:43:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T16:43:22,836 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:43:22,837 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:43:22,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:43:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T16:43:22,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T16:43:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:22,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:22,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a7c08a6b1abb4a248aa8fc992733ddd4 2024-11-24T16:43:23,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/d66c1276603341f290a1c47af76a8eca is 50, key is test_row_0/B:col10/1732466602247/Put/seqid=0 2024-11-24T16:43:23,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742455_1631 (size=12301) 2024-11-24T16:43:23,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:23,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466663083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:23,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:23,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T16:43:23,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T16:43:23,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T16:43:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T16:43:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T16:43:23,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/d66c1276603341f290a1c47af76a8eca 2024-11-24T16:43:23,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/abef6c0198294e4eb4f92dbd91600d58 is 50, key is test_row_0/C:col10/1732466602247/Put/seqid=0 2024-11-24T16:43:23,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742456_1632 (size=12301) 2024-11-24T16:43:23,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466663593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:23,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T16:43:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T16:43:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:23,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/abef6c0198294e4eb4f92dbd91600d58 2024-11-24T16:43:23,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/a7c08a6b1abb4a248aa8fc992733ddd4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a7c08a6b1abb4a248aa8fc992733ddd4 2024-11-24T16:43:23,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a7c08a6b1abb4a248aa8fc992733ddd4, entries=200, sequenceid=362, filesize=39.0 K 2024-11-24T16:43:23,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/d66c1276603341f290a1c47af76a8eca as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/d66c1276603341f290a1c47af76a8eca 2024-11-24T16:43:23,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/d66c1276603341f290a1c47af76a8eca, entries=150, sequenceid=362, filesize=12.0 K 2024-11-24T16:43:23,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/abef6c0198294e4eb4f92dbd91600d58 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/abef6c0198294e4eb4f92dbd91600d58 2024-11-24T16:43:23,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/abef6c0198294e4eb4f92dbd91600d58, entries=150, sequenceid=362, filesize=12.0 K 2024-11-24T16:43:23,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8e3281f5174968dfe4494f3e286572d3 in 1654ms, sequenceid=362, compaction requested=true 2024-11-24T16:43:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:43:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:43:23,904 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:43:23,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T16:43:23,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T16:43:23,904 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:23,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,904 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T16:43:23,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:23,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:23,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:23,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:23,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:23,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:23,907 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:23,907 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/B is initiating minor compaction (all files) 2024-11-24T16:43:23,907 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:23,907 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/B in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,907 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/A is initiating minor compaction (all files) 2024-11-24T16:43:23,907 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/610422a6d4a14cbf810d2afde9862983, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1805147183554fd4bc58fe46dcf0c6e9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/686f2a43287b4448b7e33b2789d85cbf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/d66c1276603341f290a1c47af76a8eca] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=48.5 K 2024-11-24T16:43:23,907 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/A in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,907 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ccb424a2fe344df6b70b2eb027e479fd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/90d0f5a7c36f42d498f761b38a8ca85c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2624fef9cf384f069b8a06ea535e0521, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a7c08a6b1abb4a248aa8fc992733ddd4] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=139.5 K 2024-11-24T16:43:23,907 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,907 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ccb424a2fe344df6b70b2eb027e479fd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/90d0f5a7c36f42d498f761b38a8ca85c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2624fef9cf384f069b8a06ea535e0521, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a7c08a6b1abb4a248aa8fc992733ddd4] 2024-11-24T16:43:23,908 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 610422a6d4a14cbf810d2afde9862983, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732466598664 2024-11-24T16:43:23,908 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccb424a2fe344df6b70b2eb027e479fd, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732466598664 2024-11-24T16:43:23,909 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 1805147183554fd4bc58fe46dcf0c6e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732466599388 2024-11-24T16:43:23,909 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d0f5a7c36f42d498f761b38a8ca85c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732466599382 2024-11-24T16:43:23,909 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 686f2a43287b4448b7e33b2789d85cbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732466600046 2024-11-24T16:43:23,909 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2624fef9cf384f069b8a06ea535e0521, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732466600046 2024-11-24T16:43:23,909 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting d66c1276603341f290a1c47af76a8eca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732466602243 2024-11-24T16:43:23,909 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7c08a6b1abb4a248aa8fc992733ddd4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732466602240 2024-11-24T16:43:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112424e12380756c409795d1ed1d5296eace_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466602448/Put/seqid=0 2024-11-24T16:43:23,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742457_1633 (size=12454) 2024-11-24T16:43:23,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:23,925 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112424e12380756c409795d1ed1d5296eace_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112424e12380756c409795d1ed1d5296eace_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:23,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2fccc8c226704500a017b06ec6ae0fff, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:23,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2fccc8c226704500a017b06ec6ae0fff is 175, key is test_row_0/A:col10/1732466602448/Put/seqid=0 2024-11-24T16:43:23,929 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:23,930 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124b39b56e0853847438cd19bcc71d87df5_8e3281f5174968dfe4494f3e286572d3 store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:23,930 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#B#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:23,931 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5bb9d0e3661c40378b0d5058ad03bfd8 is 50, key is test_row_0/B:col10/1732466602247/Put/seqid=0 2024-11-24T16:43:23,933 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124b39b56e0853847438cd19bcc71d87df5_8e3281f5174968dfe4494f3e286572d3, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:23,933 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b39b56e0853847438cd19bcc71d87df5_8e3281f5174968dfe4494f3e286572d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:23,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742458_1634 (size=31255) 2024-11-24T16:43:23,947 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2fccc8c226704500a017b06ec6ae0fff 2024-11-24T16:43:23,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742459_1635 (size=12915) 2024-11-24T16:43:23,955 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/5bb9d0e3661c40378b0d5058ad03bfd8 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5bb9d0e3661c40378b0d5058ad03bfd8 2024-11-24T16:43:23,959 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/B of 8e3281f5174968dfe4494f3e286572d3 into 5bb9d0e3661c40378b0d5058ad03bfd8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:23,959 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:23,959 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/B, priority=12, startTime=1732466603904; duration=0sec 2024-11-24T16:43:23,959 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:23,959 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:B 2024-11-24T16:43:23,959 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:23,960 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:23,960 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/C is initiating minor compaction (all files) 2024-11-24T16:43:23,960 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/C in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:23,960 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/294b1b2a160b44f2bb3e5f1cace69f60, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0c33830ee90443bfbc0368653f53b54e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/abef6c0198294e4eb4f92dbd91600d58] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=48.5 K 2024-11-24T16:43:23,960 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 294b1b2a160b44f2bb3e5f1cace69f60, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732466598664 2024-11-24T16:43:23,961 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 79bdbc9cd8a94a4cbf1af4ddfba5b64f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732466599388 2024-11-24T16:43:23,961 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c33830ee90443bfbc0368653f53b54e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732466600046 2024-11-24T16:43:23,962 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting abef6c0198294e4eb4f92dbd91600d58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732466602243 2024-11-24T16:43:23,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742460_1636 (size=4469) 2024-11-24T16:43:23,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/63074f6810bd4b10aaad0ed976a63e8d is 50, key is test_row_0/B:col10/1732466602448/Put/seqid=0 2024-11-24T16:43:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T16:43:23,975 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#C#compaction#537 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:23,976 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/bd8e62cac3d547a58b9475552d394ed6 is 50, key is test_row_0/C:col10/1732466602247/Put/seqid=0 2024-11-24T16:43:23,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742461_1637 (size=12915) 2024-11-24T16:43:23,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742462_1638 (size=12301) 2024-11-24T16:43:23,984 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/bd8e62cac3d547a58b9475552d394ed6 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/bd8e62cac3d547a58b9475552d394ed6 2024-11-24T16:43:23,988 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/C of 8e3281f5174968dfe4494f3e286572d3 into bd8e62cac3d547a58b9475552d394ed6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:23,988 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:23,988 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/C, priority=12, startTime=1732466603904; duration=0sec 2024-11-24T16:43:23,988 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:23,988 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:C 2024-11-24T16:43:24,368 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#A#compaction#534 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:24,368 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/806d69a469cf435bbde483e1f3abb62b is 175, key is test_row_0/A:col10/1732466602247/Put/seqid=0 2024-11-24T16:43:24,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742463_1639 (size=31869) 2024-11-24T16:43:24,385 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/63074f6810bd4b10aaad0ed976a63e8d 2024-11-24T16:43:24,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/02423e51eb2b408697d01022553eb501 is 50, key is test_row_0/C:col10/1732466602448/Put/seqid=0 2024-11-24T16:43:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742464_1640 (size=12301) 2024-11-24T16:43:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:24,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:24,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466664649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:24,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466664764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:24,777 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/806d69a469cf435bbde483e1f3abb62b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/806d69a469cf435bbde483e1f3abb62b 2024-11-24T16:43:24,780 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/A of 8e3281f5174968dfe4494f3e286572d3 into 806d69a469cf435bbde483e1f3abb62b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:24,780 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:24,780 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/A, priority=12, startTime=1732466603904; duration=0sec 2024-11-24T16:43:24,780 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:24,780 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:A 2024-11-24T16:43:24,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/02423e51eb2b408697d01022553eb501 2024-11-24T16:43:24,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/2fccc8c226704500a017b06ec6ae0fff as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2fccc8c226704500a017b06ec6ae0fff 2024-11-24T16:43:24,810 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2fccc8c226704500a017b06ec6ae0fff, entries=150, sequenceid=399, filesize=30.5 K 2024-11-24T16:43:24,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/63074f6810bd4b10aaad0ed976a63e8d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/63074f6810bd4b10aaad0ed976a63e8d 2024-11-24T16:43:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,814 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/63074f6810bd4b10aaad0ed976a63e8d, entries=150, sequenceid=399, filesize=12.0 K 2024-11-24T16:43:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/02423e51eb2b408697d01022553eb501 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/02423e51eb2b408697d01022553eb501 2024-11-24T16:43:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,818 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/02423e51eb2b408697d01022553eb501, entries=150, sequenceid=399, filesize=12.0 K 2024-11-24T16:43:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,819 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8e3281f5174968dfe4494f3e286572d3 in 915ms, sequenceid=399, compaction requested=false 2024-11-24T16:43:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:24,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-24T16:43:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-24T16:43:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-24T16:43:24,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9830 sec 2024-11-24T16:43:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9890 sec 2024-11-24T16:43:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T16:43:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:24,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T16:43:24,984 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T16:43:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-24T16:43:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T16:43:24,987 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T16:43:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,988 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T16:43:24,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T16:43:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245e9f721d5d6d475bb82375c70449225d_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466604648/Put/seqid=0 2024-11-24T16:43:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742465_1641 (size=12454) 2024-11-24T16:43:25,039 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245e9f721d5d6d475bb82375c70449225d_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245e9f721d5d6d475bb82375c70449225d_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:25,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b493dbef265b4d15b9e78045c2e7ba5d, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:25,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b493dbef265b4d15b9e78045c2e7ba5d is 175, key is test_row_0/A:col10/1732466604648/Put/seqid=0 2024-11-24T16:43:25,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742466_1642 (size=31255) 2024-11-24T16:43:25,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b493dbef265b4d15b9e78045c2e7ba5d 2024-11-24T16:43:25,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/4cf1474faf324e0391ecf2b5beb69414 is 50, key is test_row_0/B:col10/1732466604648/Put/seqid=0 2024-11-24T16:43:25,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742467_1643 (size=12301) 2024-11-24T16:43:25,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/4cf1474faf324e0391ecf2b5beb69414 2024-11-24T16:43:25,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/3c728a12eef1458882018bbbfbc1b037 is 50, key is test_row_0/C:col10/1732466604648/Put/seqid=0 2024-11-24T16:43:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T16:43:25,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742468_1644 (size=12301) 2024-11-24T16:43:25,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466605078, max timestamp=1732466605105Num uniques 859; 2024-11-24T16:43:25,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:25,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466605078, max timestamp=1732466605105Num uniques 859; 2024-11-24T16:43:25,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:25,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466605078, max timestamp=1732466605105Num uniques 859; 2024-11-24T16:43:25,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:25,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T16:43:25,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:25,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466665202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:25,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T16:43:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T16:43:25,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466665313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:25,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T16:43:25,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:25,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/3c728a12eef1458882018bbbfbc1b037 2024-11-24T16:43:25,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b493dbef265b4d15b9e78045c2e7ba5d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b493dbef265b4d15b9e78045c2e7ba5d 2024-11-24T16:43:25,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b493dbef265b4d15b9e78045c2e7ba5d, entries=150, sequenceid=413, filesize=30.5 K 2024-11-24T16:43:25,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/4cf1474faf324e0391ecf2b5beb69414 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4cf1474faf324e0391ecf2b5beb69414 2024-11-24T16:43:25,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4cf1474faf324e0391ecf2b5beb69414, entries=150, sequenceid=413, filesize=12.0 K 2024-11-24T16:43:25,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/3c728a12eef1458882018bbbfbc1b037 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/3c728a12eef1458882018bbbfbc1b037 2024-11-24T16:43:25,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/3c728a12eef1458882018bbbfbc1b037, entries=150, sequenceid=413, filesize=12.0 K 2024-11-24T16:43:25,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8e3281f5174968dfe4494f3e286572d3 in 536ms, sequenceid=413, compaction requested=true 2024-11-24T16:43:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:43:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:25,515 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:43:25,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:25,515 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:25,515 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:25,515 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/A is initiating minor compaction (all files) 2024-11-24T16:43:25,515 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:25,516 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/B is initiating minor compaction (all files) 2024-11-24T16:43:25,516 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/A in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,516 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/B in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,516 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5bb9d0e3661c40378b0d5058ad03bfd8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/63074f6810bd4b10aaad0ed976a63e8d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4cf1474faf324e0391ecf2b5beb69414] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=36.6 K 2024-11-24T16:43:25,516 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/806d69a469cf435bbde483e1f3abb62b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2fccc8c226704500a017b06ec6ae0fff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b493dbef265b4d15b9e78045c2e7ba5d] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=92.2 K 2024-11-24T16:43:25,516 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,516 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/806d69a469cf435bbde483e1f3abb62b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2fccc8c226704500a017b06ec6ae0fff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b493dbef265b4d15b9e78045c2e7ba5d] 2024-11-24T16:43:25,516 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb9d0e3661c40378b0d5058ad03bfd8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732466602243 2024-11-24T16:43:25,516 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806d69a469cf435bbde483e1f3abb62b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732466602243 2024-11-24T16:43:25,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:43:25,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:25,516 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 63074f6810bd4b10aaad0ed976a63e8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732466602419 2024-11-24T16:43:25,516 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fccc8c226704500a017b06ec6ae0fff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732466602419 2024-11-24T16:43:25,517 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf1474faf324e0391ecf2b5beb69414, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732466604642 2024-11-24T16:43:25,517 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b493dbef265b4d15b9e78045c2e7ba5d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732466604642 2024-11-24T16:43:25,523 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:25,525 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#B#compaction#543 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:25,526 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/c88627a148ab46489cedb8c963895623 is 50, key is test_row_0/B:col10/1732466604648/Put/seqid=0 2024-11-24T16:43:25,527 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240cf81e79e08e49c89dc8cc8ef1d30bd1_8e3281f5174968dfe4494f3e286572d3 store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:25,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T16:43:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:25,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T16:43:25,529 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240cf81e79e08e49c89dc8cc8ef1d30bd1_8e3281f5174968dfe4494f3e286572d3, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:25,529 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240cf81e79e08e49c89dc8cc8ef1d30bd1_8e3281f5174968dfe4494f3e286572d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:25,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742469_1645 (size=13017) 2024-11-24T16:43:25,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742470_1646 (size=4469) 2024-11-24T16:43:25,538 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/c88627a148ab46489cedb8c963895623 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/c88627a148ab46489cedb8c963895623 2024-11-24T16:43:25,542 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/B of 8e3281f5174968dfe4494f3e286572d3 into c88627a148ab46489cedb8c963895623(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:25,542 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:25,543 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/B, priority=13, startTime=1732466605515; duration=0sec 2024-11-24T16:43:25,543 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:25,543 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:B 2024-11-24T16:43:25,543 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T16:43:25,544 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T16:43:25,544 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/C is initiating minor compaction (all files) 2024-11-24T16:43:25,544 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/C in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,544 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/bd8e62cac3d547a58b9475552d394ed6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/02423e51eb2b408697d01022553eb501, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/3c728a12eef1458882018bbbfbc1b037] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=36.6 K 2024-11-24T16:43:25,544 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8e62cac3d547a58b9475552d394ed6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732466602243 2024-11-24T16:43:25,544 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 02423e51eb2b408697d01022553eb501, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732466602419 2024-11-24T16:43:25,544 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c728a12eef1458882018bbbfbc1b037, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732466604642 2024-11-24T16:43:25,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112452b5528df27242bda031058469add30c_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466605187/Put/seqid=0 2024-11-24T16:43:25,552 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#C#compaction#545 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:25,553 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/b663a6e225984dfca85c5d6d02794d5a is 50, key is test_row_0/C:col10/1732466604648/Put/seqid=0 2024-11-24T16:43:25,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742471_1647 (size=14994) 2024-11-24T16:43:25,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742472_1648 (size=13017) 2024-11-24T16:43:25,564 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/b663a6e225984dfca85c5d6d02794d5a as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/b663a6e225984dfca85c5d6d02794d5a 2024-11-24T16:43:25,569 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/C of 8e3281f5174968dfe4494f3e286572d3 into b663a6e225984dfca85c5d6d02794d5a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:25,569 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:25,569 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/C, priority=13, startTime=1732466605516; duration=0sec 2024-11-24T16:43:25,569 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:25,569 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:C 2024-11-24T16:43:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 338 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466665574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:25,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T16:43:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T16:43:25,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466665683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:25,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T16:43:25,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:25,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466665894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:25,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:25,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T16:43:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:25,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T16:43:25,936 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#A#compaction#542 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:25,936 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/48e01f09fe174eeea01a162b437a77fa is 175, key is test_row_0/A:col10/1732466604648/Put/seqid=0 2024-11-24T16:43:25,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742473_1649 (size=31971) 2024-11-24T16:43:25,950 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/48e01f09fe174eeea01a162b437a77fa as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/48e01f09fe174eeea01a162b437a77fa 2024-11-24T16:43:25,954 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/A of 8e3281f5174968dfe4494f3e286572d3 into 48e01f09fe174eeea01a162b437a77fa(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:25,954 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:25,955 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/A, priority=13, startTime=1732466605515; duration=0sec 2024-11-24T16:43:25,955 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:25,955 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:A 2024-11-24T16:43:25,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,962 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112452b5528df27242bda031058469add30c_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112452b5528df27242bda031058469add30c_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:25,963 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b7c1afe592ee439e9f3226800dd46d57, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:25,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b7c1afe592ee439e9f3226800dd46d57 is 175, key is test_row_0/A:col10/1732466605187/Put/seqid=0 2024-11-24T16:43:25,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742474_1650 (size=39949) 2024-11-24T16:43:25,968 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=450, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b7c1afe592ee439e9f3226800dd46d57 2024-11-24T16:43:25,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/b0b6e4c5de72484ab14872911218ab8b is 50, key is test_row_0/B:col10/1732466605187/Put/seqid=0 2024-11-24T16:43:25,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742475_1651 (size=12301) 2024-11-24T16:43:25,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/b0b6e4c5de72484ab14872911218ab8b 2024-11-24T16:43:25,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/082f0db458ea47488992f921c32fcba4 is 50, key is test_row_0/C:col10/1732466605187/Put/seqid=0 2024-11-24T16:43:25,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742476_1652 (size=12301) 2024-11-24T16:43:25,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/082f0db458ea47488992f921c32fcba4 2024-11-24T16:43:25,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/b7c1afe592ee439e9f3226800dd46d57 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b7c1afe592ee439e9f3226800dd46d57 2024-11-24T16:43:25,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b7c1afe592ee439e9f3226800dd46d57, entries=200, sequenceid=450, filesize=39.0 K 2024-11-24T16:43:25,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/b0b6e4c5de72484ab14872911218ab8b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/b0b6e4c5de72484ab14872911218ab8b 2024-11-24T16:43:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:25,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/b0b6e4c5de72484ab14872911218ab8b, entries=150, sequenceid=450, filesize=12.0 K 2024-11-24T16:43:26,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/082f0db458ea47488992f921c32fcba4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/082f0db458ea47488992f921c32fcba4 2024-11-24T16:43:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/082f0db458ea47488992f921c32fcba4, entries=150, sequenceid=450, filesize=12.0 K 2024-11-24T16:43:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8e3281f5174968dfe4494f3e286572d3 in 476ms, sequenceid=450, compaction requested=false 2024-11-24T16:43:26,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T16:43:26,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,058 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T16:43:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:26,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e062fd48de8d40be887e398d474fe3c9_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466605566/Put/seqid=0 2024-11-24T16:43:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742477_1653 (size=9914) 2024-11-24T16:43:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T16:43:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. as already flushing 2024-11-24T16:43:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466606219, max timestamp=1732466606270Num uniques 859; 2024-11-24T16:43:26,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466606219, max timestamp=1732466606270Num uniques 859; 2024-11-24T16:43:26,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:26,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:26,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732466606219, max timestamp=1732466606270Num uniques 859; 2024-11-24T16:43:26,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/e335232f7491:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T16:43:26,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 375 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466666358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:26,476 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e062fd48de8d40be887e398d474fe3c9_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e062fd48de8d40be887e398d474fe3c9_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:26,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/ac0757d420d84c07b405ccdabdc58421, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:26,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/ac0757d420d84c07b405ccdabdc58421 is 175, key is test_row_0/A:col10/1732466605566/Put/seqid=0 2024-11-24T16:43:26,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 376 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466666474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:26,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742478_1654 (size=22561) 2024-11-24T16:43:26,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 377 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466666684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:26,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:26,891 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=463, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/ac0757d420d84c07b405ccdabdc58421 2024-11-24T16:43:26,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/dbdb12444a9045328d795dba4110aca5 is 50, key is test_row_0/B:col10/1732466605566/Put/seqid=0 2024-11-24T16:43:26,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742479_1655 (size=9857) 2024-11-24T16:43:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 378 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466666993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T16:43:27,301 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/dbdb12444a9045328d795dba4110aca5 2024-11-24T16:43:27,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/e093f4a908e044e3b8645b62a6fd09f4 is 50, key is test_row_0/C:col10/1732466605566/Put/seqid=0 2024-11-24T16:43:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742480_1656 (size=9857) 2024-11-24T16:43:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 379 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39648 deadline: 1732466667504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:27,710 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/e093f4a908e044e3b8645b62a6fd09f4 2024-11-24T16:43:27,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/ac0757d420d84c07b405ccdabdc58421 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ac0757d420d84c07b405ccdabdc58421 2024-11-24T16:43:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T16:43:27,717 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ac0757d420d84c07b405ccdabdc58421, entries=100, sequenceid=463, filesize=22.0 K 2024-11-24T16:43:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39664 deadline: 1732466667714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 2024-11-24T16:43:27,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/dbdb12444a9045328d795dba4110aca5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/dbdb12444a9045328d795dba4110aca5 2024-11-24T16:43:27,718 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to e335232f7491,33607,1732466431346 for 'test_row_1' in 8e3281f5174968dfe4494f3e286572d3 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T16:43:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 , the old value is region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T16:43:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., hostname=e335232f7491,33607,1732466431346, seqNum=5 because the exception is null or not the one we care about 2024-11-24T16:43:27,721 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/dbdb12444a9045328d795dba4110aca5, entries=100, sequenceid=463, filesize=9.6 K 2024-11-24T16:43:27,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/e093f4a908e044e3b8645b62a6fd09f4 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e093f4a908e044e3b8645b62a6fd09f4 2024-11-24T16:43:27,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e093f4a908e044e3b8645b62a6fd09f4, entries=100, sequenceid=463, filesize=9.6 K 2024-11-24T16:43:27,725 INFO [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8e3281f5174968dfe4494f3e286572d3 in 1667ms, sequenceid=463, compaction requested=true 2024-11-24T16:43:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e335232f7491:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-24T16:43:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-24T16:43:27,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-24T16:43:27,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7390 sec 2024-11-24T16:43:27,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.7440 sec 2024-11-24T16:43:28,052 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-11-24T16:43:28,052 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:28,052 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,052 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:28,052 INFO [Thread-2505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2505. 2024-11-24T16:43:28,052 DEBUG [Thread-2505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:28,052 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,052 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:28,053 INFO [Thread-2499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2499. 2024-11-24T16:43:28,053 DEBUG [Thread-2499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:28,053 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,053 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,053 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:28,055 INFO [Thread-2501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2501. 2024-11-24T16:43:28,056 DEBUG [Thread-2501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:28,056 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,056 INFO [Thread-2503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2503. 2024-11-24T16:43:28,056 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,056 DEBUG [Thread-2503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:28,056 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:28,056 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33607 {}] regionserver.HRegion(8855): Flush requested on 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:28,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-24T16:43:28,526 INFO [Thread-2488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2488. 2024-11-24T16:43:28,526 DEBUG [Thread-2488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:28,526 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:43:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:28,526 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:43:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:28,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:28,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T16:43:28,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244ad8d58f37be4328bb954fb83218992d_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466608524/Put/seqid=0 2024-11-24T16:43:28,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742481_1657 (size=12454) 2024-11-24T16:43:28,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:28,937 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244ad8d58f37be4328bb954fb83218992d_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ad8d58f37be4328bb954fb83218992d_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:28,937 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/9744cfc2aa344c0e8f5d634b3d1c9d3c, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:28,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/9744cfc2aa344c0e8f5d634b3d1c9d3c is 175, key is test_row_0/A:col10/1732466608524/Put/seqid=0 2024-11-24T16:43:28,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742482_1658 (size=31255) 2024-11-24T16:43:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T16:43:29,164 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:29,336 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T16:43:29,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=501, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/9744cfc2aa344c0e8f5d634b3d1c9d3c 2024-11-24T16:43:29,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/6e299afd2d9a4a0a8b6db29f7a147604 is 50, key is test_row_0/B:col10/1732466608524/Put/seqid=0 2024-11-24T16:43:29,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742483_1659 (size=12301) 2024-11-24T16:43:29,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/6e299afd2d9a4a0a8b6db29f7a147604 2024-11-24T16:43:29,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/36f79aef759f4f1093aa9d5ce78504b5 is 50, key is test_row_0/C:col10/1732466608524/Put/seqid=0 2024-11-24T16:43:29,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742484_1660 (size=12301) 2024-11-24T16:43:30,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/36f79aef759f4f1093aa9d5ce78504b5 2024-11-24T16:43:30,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/9744cfc2aa344c0e8f5d634b3d1c9d3c as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/9744cfc2aa344c0e8f5d634b3d1c9d3c 2024-11-24T16:43:30,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/9744cfc2aa344c0e8f5d634b3d1c9d3c, entries=150, sequenceid=501, filesize=30.5 K 2024-11-24T16:43:30,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/6e299afd2d9a4a0a8b6db29f7a147604 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/6e299afd2d9a4a0a8b6db29f7a147604 2024-11-24T16:43:30,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/6e299afd2d9a4a0a8b6db29f7a147604, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T16:43:30,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/36f79aef759f4f1093aa9d5ce78504b5 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/36f79aef759f4f1093aa9d5ce78504b5 2024-11-24T16:43:30,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/36f79aef759f4f1093aa9d5ce78504b5, entries=150, sequenceid=501, filesize=12.0 K 2024-11-24T16:43:30,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=0 B/0 for 8e3281f5174968dfe4494f3e286572d3 in 1645ms, sequenceid=501, compaction requested=true 2024-11-24T16:43:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T16:43:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T16:43:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:30,170 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8e3281f5174968dfe4494f3e286572d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T16:43:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:30,170 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:30,171 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125736 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:30,171 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:30,171 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/B is initiating minor compaction (all files) 2024-11-24T16:43:30,171 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/A is initiating minor compaction (all files) 2024-11-24T16:43:30,171 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/B in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:30,171 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/A in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:30,171 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/c88627a148ab46489cedb8c963895623, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/b0b6e4c5de72484ab14872911218ab8b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/dbdb12444a9045328d795dba4110aca5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/6e299afd2d9a4a0a8b6db29f7a147604] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=46.4 K 2024-11-24T16:43:30,171 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/48e01f09fe174eeea01a162b437a77fa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b7c1afe592ee439e9f3226800dd46d57, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ac0757d420d84c07b405ccdabdc58421, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/9744cfc2aa344c0e8f5d634b3d1c9d3c] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=122.8 K 2024-11-24T16:43:30,171 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:30,171 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. files: [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/48e01f09fe174eeea01a162b437a77fa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b7c1afe592ee439e9f3226800dd46d57, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ac0757d420d84c07b405ccdabdc58421, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/9744cfc2aa344c0e8f5d634b3d1c9d3c] 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting c88627a148ab46489cedb8c963895623, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732466604642 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e01f09fe174eeea01a162b437a77fa, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732466604642 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting b0b6e4c5de72484ab14872911218ab8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732466605187 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7c1afe592ee439e9f3226800dd46d57, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732466605187 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting dbdb12444a9045328d795dba4110aca5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732466605566 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac0757d420d84c07b405ccdabdc58421, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732466605566 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e299afd2d9a4a0a8b6db29f7a147604, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732466606337 2024-11-24T16:43:30,172 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9744cfc2aa344c0e8f5d634b3d1c9d3c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732466606337 2024-11-24T16:43:30,177 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:30,178 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#B#compaction#554 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:30,178 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112462094bda6b6243d3af0d14009d4f0079_8e3281f5174968dfe4494f3e286572d3 store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:30,179 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/14027f6917c740af870dd14ca9e7c02d is 50, key is test_row_0/B:col10/1732466608524/Put/seqid=0 2024-11-24T16:43:30,181 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112462094bda6b6243d3af0d14009d4f0079_8e3281f5174968dfe4494f3e286572d3, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:30,181 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112462094bda6b6243d3af0d14009d4f0079_8e3281f5174968dfe4494f3e286572d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:30,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742485_1661 (size=13153) 2024-11-24T16:43:30,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742486_1662 (size=4469) 2024-11-24T16:43:30,185 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#A#compaction#555 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:30,185 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/348c1e3b43f24117917210ed7e7a4e74 is 175, key is test_row_0/A:col10/1732466608524/Put/seqid=0 2024-11-24T16:43:30,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742487_1663 (size=32107) 2024-11-24T16:43:30,190 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/348c1e3b43f24117917210ed7e7a4e74 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/348c1e3b43f24117917210ed7e7a4e74 2024-11-24T16:43:30,193 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/A of 8e3281f5174968dfe4494f3e286572d3 into 348c1e3b43f24117917210ed7e7a4e74(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:30,193 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:30,193 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/A, priority=12, startTime=1732466610170; duration=0sec 2024-11-24T16:43:30,193 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T16:43:30,193 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:A 2024-11-24T16:43:30,193 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T16:43:30,194 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T16:43:30,194 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1541): 8e3281f5174968dfe4494f3e286572d3/C is initiating minor compaction (all files) 2024-11-24T16:43:30,194 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8e3281f5174968dfe4494f3e286572d3/C in TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:30,194 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/b663a6e225984dfca85c5d6d02794d5a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/082f0db458ea47488992f921c32fcba4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e093f4a908e044e3b8645b62a6fd09f4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/36f79aef759f4f1093aa9d5ce78504b5] into tmpdir=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp, totalSize=46.4 K 2024-11-24T16:43:30,195 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting b663a6e225984dfca85c5d6d02794d5a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732466604642 2024-11-24T16:43:30,195 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 082f0db458ea47488992f921c32fcba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732466605187 2024-11-24T16:43:30,195 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting e093f4a908e044e3b8645b62a6fd09f4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732466605566 2024-11-24T16:43:30,195 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36f79aef759f4f1093aa9d5ce78504b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1732466606337 2024-11-24T16:43:30,200 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8e3281f5174968dfe4494f3e286572d3#C#compaction#556 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T16:43:30,201 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/16f0df51ba774611881355197d066605 is 50, key is test_row_0/C:col10/1732466608524/Put/seqid=0 2024-11-24T16:43:30,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742488_1664 (size=13153) 2024-11-24T16:43:30,586 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/14027f6917c740af870dd14ca9e7c02d as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/14027f6917c740af870dd14ca9e7c02d 2024-11-24T16:43:30,589 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/B of 8e3281f5174968dfe4494f3e286572d3 into 14027f6917c740af870dd14ca9e7c02d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:30,590 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:30,590 INFO [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/B, priority=12, startTime=1732466610170; duration=0sec 2024-11-24T16:43:30,590 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:30,590 DEBUG [RS:0;e335232f7491:33607-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:B 2024-11-24T16:43:30,606 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/16f0df51ba774611881355197d066605 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/16f0df51ba774611881355197d066605 2024-11-24T16:43:30,609 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8e3281f5174968dfe4494f3e286572d3/C of 8e3281f5174968dfe4494f3e286572d3 into 16f0df51ba774611881355197d066605(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T16:43:30,609 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8e3281f5174968dfe4494f3e286572d3: 2024-11-24T16:43:30,609 INFO [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., storeName=8e3281f5174968dfe4494f3e286572d3/C, priority=12, startTime=1732466610170; duration=0sec 2024-11-24T16:43:30,609 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T16:43:30,609 DEBUG [RS:0;e335232f7491:33607-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8e3281f5174968dfe4494f3e286572d3:C 2024-11-24T16:43:31,125 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-11-24T16:43:31,125 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:31,125 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:31,125 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:31,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:31,145 INFO [Thread-2490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2490. 2024-11-24T16:43:31,145 DEBUG [Thread-2490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:31,145 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:31,146 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:31,146 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:31,175 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-11-24T16:43:31,175 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:31,175 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:31,175 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:31,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:37,775 INFO [Thread-2492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2492. 2024-11-24T16:43:37,775 DEBUG [Thread-2492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T16:43:37,775 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:37,775 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:37,775 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 321 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2733 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2606 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2612 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2742 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2612 2024-11-24T16:43:37,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T16:43:37,776 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:43:37,776 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:43:37,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:37,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:37,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:37,776 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T16:43:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T16:43:37,779 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466617779"}]},"ts":"1732466617779"} 2024-11-24T16:43:37,780 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T16:43:37,780 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T16:43:37,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T16:43:37,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, UNASSIGN}] 2024-11-24T16:43:37,782 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, UNASSIGN 2024-11-24T16:43:37,783 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=CLOSING, regionLocation=e335232f7491,33607,1732466431346 2024-11-24T16:43:37,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, UNASSIGN because future has completed 2024-11-24T16:43:37,784 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T16:43:37,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346}] 2024-11-24T16:43:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T16:43:37,936 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:37,936 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T16:43:37,936 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 8e3281f5174968dfe4494f3e286572d3, disabling compactions & flushes 2024-11-24T16:43:37,936 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:37,936 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:37,936 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. after waiting 0 ms 2024-11-24T16:43:37,936 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:37,936 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 8e3281f5174968dfe4494f3e286572d3 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T16:43:37,937 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=A 2024-11-24T16:43:37,937 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:37,937 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=B 2024-11-24T16:43:37,937 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:37,937 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8e3281f5174968dfe4494f3e286572d3, store=C 2024-11-24T16:43:37,937 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T16:43:37,941 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bf47c32ab7e64d4fb532d95e3761845c_8e3281f5174968dfe4494f3e286572d3 is 50, key is test_row_0/A:col10/1732466611144/Put/seqid=0 2024-11-24T16:43:37,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742489_1665 (size=9914) 2024-11-24T16:43:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T16:43:38,345 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T16:43:38,348 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bf47c32ab7e64d4fb532d95e3761845c_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bf47c32ab7e64d4fb532d95e3761845c_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:38,348 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/7385a13cc14a4ea08101623970f71914, store: [table=TestAcidGuarantees family=A region=8e3281f5174968dfe4494f3e286572d3] 2024-11-24T16:43:38,349 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/7385a13cc14a4ea08101623970f71914 is 175, key is test_row_0/A:col10/1732466611144/Put/seqid=0 2024-11-24T16:43:38,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742490_1666 (size=22561) 2024-11-24T16:43:38,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T16:43:38,752 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=511, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/7385a13cc14a4ea08101623970f71914 2024-11-24T16:43:38,757 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/a6a3820fd488402695f51fe0c817def7 is 50, key is test_row_0/B:col10/1732466611144/Put/seqid=0 2024-11-24T16:43:38,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742491_1667 (size=9857) 2024-11-24T16:43:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T16:43:39,161 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/a6a3820fd488402695f51fe0c817def7 2024-11-24T16:43:39,166 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/0b33144898a746d1b8b1cbc8f6a0ca4b is 50, key is test_row_0/C:col10/1732466611144/Put/seqid=0 2024-11-24T16:43:39,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742492_1668 (size=9857) 2024-11-24T16:43:39,569 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/0b33144898a746d1b8b1cbc8f6a0ca4b 2024-11-24T16:43:39,573 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/A/7385a13cc14a4ea08101623970f71914 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/7385a13cc14a4ea08101623970f71914 2024-11-24T16:43:39,575 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/7385a13cc14a4ea08101623970f71914, entries=100, sequenceid=511, filesize=22.0 K 2024-11-24T16:43:39,576 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/B/a6a3820fd488402695f51fe0c817def7 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/a6a3820fd488402695f51fe0c817def7 2024-11-24T16:43:39,579 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/a6a3820fd488402695f51fe0c817def7, entries=100, sequenceid=511, filesize=9.6 K 2024-11-24T16:43:39,579 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/.tmp/C/0b33144898a746d1b8b1cbc8f6a0ca4b as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0b33144898a746d1b8b1cbc8f6a0ca4b 2024-11-24T16:43:39,582 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0b33144898a746d1b8b1cbc8f6a0ca4b, entries=100, sequenceid=511, filesize=9.6 K 2024-11-24T16:43:39,582 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8e3281f5174968dfe4494f3e286572d3 in 1646ms, sequenceid=511, compaction requested=false 2024-11-24T16:43:39,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/db926aa5939b4f668c5eba4a69c15867, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a9bc7afff8c549c6ad897c21f2ef94cb, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/540d037bc9fe43e3accddd55235aae6b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5b9e2ae4e9294376b1b364ffc57157be, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/4668e71f205543be8ddd6332c2e6b132, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b49f362a336245bb81dd97cb3df44a52, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/22d5e037f0344683a16a52011ac82182, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a850da2d5d904e55b6f2367e545e659b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/16fd28dc382d436fbdef692853db7f8d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/cc1c87e8ebe545e78bb9ff15260515c1, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/0c50188748144be1be816648a5bb769d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5e9a2f1071d94827a3c578f316bc2c7b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2a5b8a60fce34f5082503447705eb4ba, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ccb424a2fe344df6b70b2eb027e479fd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/655b101ca6fb47a4b216737136a52f24, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/90d0f5a7c36f42d498f761b38a8ca85c, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2624fef9cf384f069b8a06ea535e0521, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a7c08a6b1abb4a248aa8fc992733ddd4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/806d69a469cf435bbde483e1f3abb62b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2fccc8c226704500a017b06ec6ae0fff, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/48e01f09fe174eeea01a162b437a77fa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b493dbef265b4d15b9e78045c2e7ba5d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b7c1afe592ee439e9f3226800dd46d57, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ac0757d420d84c07b405ccdabdc58421, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/9744cfc2aa344c0e8f5d634b3d1c9d3c] to archive 2024-11-24T16:43:39,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:43:39,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/db926aa5939b4f668c5eba4a69c15867 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/db926aa5939b4f668c5eba4a69c15867 2024-11-24T16:43:39,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a9bc7afff8c549c6ad897c21f2ef94cb to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a9bc7afff8c549c6ad897c21f2ef94cb 2024-11-24T16:43:39,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/540d037bc9fe43e3accddd55235aae6b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/540d037bc9fe43e3accddd55235aae6b 2024-11-24T16:43:39,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5b9e2ae4e9294376b1b364ffc57157be to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5b9e2ae4e9294376b1b364ffc57157be 2024-11-24T16:43:39,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/4668e71f205543be8ddd6332c2e6b132 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/4668e71f205543be8ddd6332c2e6b132 2024-11-24T16:43:39,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b49f362a336245bb81dd97cb3df44a52 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b49f362a336245bb81dd97cb3df44a52 2024-11-24T16:43:39,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/22d5e037f0344683a16a52011ac82182 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/22d5e037f0344683a16a52011ac82182 2024-11-24T16:43:39,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a850da2d5d904e55b6f2367e545e659b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a850da2d5d904e55b6f2367e545e659b 2024-11-24T16:43:39,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/16fd28dc382d436fbdef692853db7f8d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/16fd28dc382d436fbdef692853db7f8d 2024-11-24T16:43:39,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/cc1c87e8ebe545e78bb9ff15260515c1 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/cc1c87e8ebe545e78bb9ff15260515c1 2024-11-24T16:43:39,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/0c50188748144be1be816648a5bb769d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/0c50188748144be1be816648a5bb769d 2024-11-24T16:43:39,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5e9a2f1071d94827a3c578f316bc2c7b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/5e9a2f1071d94827a3c578f316bc2c7b 2024-11-24T16:43:39,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2a5b8a60fce34f5082503447705eb4ba to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2a5b8a60fce34f5082503447705eb4ba 2024-11-24T16:43:39,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ccb424a2fe344df6b70b2eb027e479fd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ccb424a2fe344df6b70b2eb027e479fd 2024-11-24T16:43:39,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/655b101ca6fb47a4b216737136a52f24 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/655b101ca6fb47a4b216737136a52f24 2024-11-24T16:43:39,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/90d0f5a7c36f42d498f761b38a8ca85c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/90d0f5a7c36f42d498f761b38a8ca85c 2024-11-24T16:43:39,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2624fef9cf384f069b8a06ea535e0521 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2624fef9cf384f069b8a06ea535e0521 2024-11-24T16:43:39,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a7c08a6b1abb4a248aa8fc992733ddd4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/a7c08a6b1abb4a248aa8fc992733ddd4 2024-11-24T16:43:39,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/806d69a469cf435bbde483e1f3abb62b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/806d69a469cf435bbde483e1f3abb62b 2024-11-24T16:43:39,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2fccc8c226704500a017b06ec6ae0fff to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/2fccc8c226704500a017b06ec6ae0fff 2024-11-24T16:43:39,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/48e01f09fe174eeea01a162b437a77fa to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/48e01f09fe174eeea01a162b437a77fa 2024-11-24T16:43:39,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b493dbef265b4d15b9e78045c2e7ba5d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b493dbef265b4d15b9e78045c2e7ba5d 2024-11-24T16:43:39,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b7c1afe592ee439e9f3226800dd46d57 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/b7c1afe592ee439e9f3226800dd46d57 2024-11-24T16:43:39,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ac0757d420d84c07b405ccdabdc58421 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/ac0757d420d84c07b405ccdabdc58421 2024-11-24T16:43:39,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/9744cfc2aa344c0e8f5d634b3d1c9d3c to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/9744cfc2aa344c0e8f5d634b3d1c9d3c 2024-11-24T16:43:39,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1eb1f0ae1ac6456383be5a08580a704e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e1cf10ee84234b96bd755d75d74f6589, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4e39aa5230134c4db03ec076d25fbbcd, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fb53624bdfa94eea94c02a6b69a7dba0, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/aba337f956a1474a9c9c838946490352, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e37a7b531f3b46b688495c484867fd1a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/50893fecde7a4269b9ce6339611bcdd8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/8eae09d69e824d02bfd7126f8c420798, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fdfd1e52c44649a980da90f1a68e4c00, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/de1ec78bc55f470d94c31a0f5c89bbdc, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/bcbe4f04271748d2a62d616f80822c29, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5e6a9e548e084fa9baaee606f0830997, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/9b831dc3c279446c9bc2e6c8801c0461, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/610422a6d4a14cbf810d2afde9862983, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5cd0a1da6aa74a2fbd5e34a428cc333e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1805147183554fd4bc58fe46dcf0c6e9, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/686f2a43287b4448b7e33b2789d85cbf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5bb9d0e3661c40378b0d5058ad03bfd8, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/d66c1276603341f290a1c47af76a8eca, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/63074f6810bd4b10aaad0ed976a63e8d, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/c88627a148ab46489cedb8c963895623, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4cf1474faf324e0391ecf2b5beb69414, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/b0b6e4c5de72484ab14872911218ab8b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/dbdb12444a9045328d795dba4110aca5, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/6e299afd2d9a4a0a8b6db29f7a147604] to archive 2024-11-24T16:43:39,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:43:39,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1eb1f0ae1ac6456383be5a08580a704e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1eb1f0ae1ac6456383be5a08580a704e 2024-11-24T16:43:39,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e1cf10ee84234b96bd755d75d74f6589 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e1cf10ee84234b96bd755d75d74f6589 2024-11-24T16:43:39,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4e39aa5230134c4db03ec076d25fbbcd to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4e39aa5230134c4db03ec076d25fbbcd 2024-11-24T16:43:39,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fb53624bdfa94eea94c02a6b69a7dba0 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fb53624bdfa94eea94c02a6b69a7dba0 2024-11-24T16:43:39,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/aba337f956a1474a9c9c838946490352 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/aba337f956a1474a9c9c838946490352 2024-11-24T16:43:39,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e37a7b531f3b46b688495c484867fd1a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/e37a7b531f3b46b688495c484867fd1a 2024-11-24T16:43:39,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/50893fecde7a4269b9ce6339611bcdd8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/50893fecde7a4269b9ce6339611bcdd8 2024-11-24T16:43:39,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/8eae09d69e824d02bfd7126f8c420798 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/8eae09d69e824d02bfd7126f8c420798 2024-11-24T16:43:39,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fdfd1e52c44649a980da90f1a68e4c00 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/fdfd1e52c44649a980da90f1a68e4c00 2024-11-24T16:43:39,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/de1ec78bc55f470d94c31a0f5c89bbdc to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/de1ec78bc55f470d94c31a0f5c89bbdc 2024-11-24T16:43:39,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/bcbe4f04271748d2a62d616f80822c29 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/bcbe4f04271748d2a62d616f80822c29 2024-11-24T16:43:39,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5e6a9e548e084fa9baaee606f0830997 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5e6a9e548e084fa9baaee606f0830997 2024-11-24T16:43:39,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/9b831dc3c279446c9bc2e6c8801c0461 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/9b831dc3c279446c9bc2e6c8801c0461 2024-11-24T16:43:39,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/610422a6d4a14cbf810d2afde9862983 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/610422a6d4a14cbf810d2afde9862983 2024-11-24T16:43:39,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5cd0a1da6aa74a2fbd5e34a428cc333e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5cd0a1da6aa74a2fbd5e34a428cc333e 2024-11-24T16:43:39,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1805147183554fd4bc58fe46dcf0c6e9 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/1805147183554fd4bc58fe46dcf0c6e9 2024-11-24T16:43:39,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/686f2a43287b4448b7e33b2789d85cbf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/686f2a43287b4448b7e33b2789d85cbf 2024-11-24T16:43:39,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5bb9d0e3661c40378b0d5058ad03bfd8 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/5bb9d0e3661c40378b0d5058ad03bfd8 2024-11-24T16:43:39,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/d66c1276603341f290a1c47af76a8eca to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/d66c1276603341f290a1c47af76a8eca 2024-11-24T16:43:39,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/63074f6810bd4b10aaad0ed976a63e8d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/63074f6810bd4b10aaad0ed976a63e8d 2024-11-24T16:43:39,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/c88627a148ab46489cedb8c963895623 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/c88627a148ab46489cedb8c963895623 2024-11-24T16:43:39,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4cf1474faf324e0391ecf2b5beb69414 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/4cf1474faf324e0391ecf2b5beb69414 2024-11-24T16:43:39,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/b0b6e4c5de72484ab14872911218ab8b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/b0b6e4c5de72484ab14872911218ab8b 2024-11-24T16:43:39,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/dbdb12444a9045328d795dba4110aca5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/dbdb12444a9045328d795dba4110aca5 2024-11-24T16:43:39,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/6e299afd2d9a4a0a8b6db29f7a147604 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/6e299afd2d9a4a0a8b6db29f7a147604 2024-11-24T16:43:39,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/608f426f1802480b9fe0fc30f35062b6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/4dd8f85b6b7f4661beff8e15c0035e26, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/56919711ece140a99494481263a114ed, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e35f4574058e41c6a8e4f192e97e4e2e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/598bcb2761a742b0bfe5aae9ca25b8a4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/72bf1ab85e474ca097292597e4df1243, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/db1c0296c81448a7914177680e3047e4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/765f0a9482994f40956b040d838e4a2b, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/5fcc90b9c51941da8a05d2d5669cf2aa, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/15125faa8f8b4defa41e39ddad2e3a44, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/1551462958ec4286ba6ccf6ed5076f7a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/f11af04c71084358ac4155f52a1480af, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/ef1d653358f547eab1ba17082a586784, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/294b1b2a160b44f2bb3e5f1cace69f60, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/8db8a313757e4285a9587eda8ad04caf, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0c33830ee90443bfbc0368653f53b54e, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/bd8e62cac3d547a58b9475552d394ed6, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/abef6c0198294e4eb4f92dbd91600d58, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/02423e51eb2b408697d01022553eb501, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/b663a6e225984dfca85c5d6d02794d5a, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/3c728a12eef1458882018bbbfbc1b037, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/082f0db458ea47488992f921c32fcba4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e093f4a908e044e3b8645b62a6fd09f4, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/36f79aef759f4f1093aa9d5ce78504b5] to archive 2024-11-24T16:43:39,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T16:43:39,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/608f426f1802480b9fe0fc30f35062b6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/608f426f1802480b9fe0fc30f35062b6 2024-11-24T16:43:39,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/4dd8f85b6b7f4661beff8e15c0035e26 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/4dd8f85b6b7f4661beff8e15c0035e26 2024-11-24T16:43:39,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/56919711ece140a99494481263a114ed to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/56919711ece140a99494481263a114ed 2024-11-24T16:43:39,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e35f4574058e41c6a8e4f192e97e4e2e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e35f4574058e41c6a8e4f192e97e4e2e 2024-11-24T16:43:39,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/598bcb2761a742b0bfe5aae9ca25b8a4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/598bcb2761a742b0bfe5aae9ca25b8a4 2024-11-24T16:43:39,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/72bf1ab85e474ca097292597e4df1243 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/72bf1ab85e474ca097292597e4df1243 2024-11-24T16:43:39,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/db1c0296c81448a7914177680e3047e4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/db1c0296c81448a7914177680e3047e4 2024-11-24T16:43:39,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/765f0a9482994f40956b040d838e4a2b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/765f0a9482994f40956b040d838e4a2b 2024-11-24T16:43:39,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/5fcc90b9c51941da8a05d2d5669cf2aa to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/5fcc90b9c51941da8a05d2d5669cf2aa 2024-11-24T16:43:39,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/15125faa8f8b4defa41e39ddad2e3a44 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/15125faa8f8b4defa41e39ddad2e3a44 2024-11-24T16:43:39,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/1551462958ec4286ba6ccf6ed5076f7a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/1551462958ec4286ba6ccf6ed5076f7a 2024-11-24T16:43:39,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/f11af04c71084358ac4155f52a1480af to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/f11af04c71084358ac4155f52a1480af 2024-11-24T16:43:39,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/ef1d653358f547eab1ba17082a586784 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/ef1d653358f547eab1ba17082a586784 2024-11-24T16:43:39,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/294b1b2a160b44f2bb3e5f1cace69f60 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/294b1b2a160b44f2bb3e5f1cace69f60 2024-11-24T16:43:39,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/8db8a313757e4285a9587eda8ad04caf to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/8db8a313757e4285a9587eda8ad04caf 2024-11-24T16:43:39,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/79bdbc9cd8a94a4cbf1af4ddfba5b64f 2024-11-24T16:43:39,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0c33830ee90443bfbc0368653f53b54e to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0c33830ee90443bfbc0368653f53b54e 2024-11-24T16:43:39,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/bd8e62cac3d547a58b9475552d394ed6 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/bd8e62cac3d547a58b9475552d394ed6 2024-11-24T16:43:39,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/abef6c0198294e4eb4f92dbd91600d58 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/abef6c0198294e4eb4f92dbd91600d58 2024-11-24T16:43:39,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/02423e51eb2b408697d01022553eb501 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/02423e51eb2b408697d01022553eb501 2024-11-24T16:43:39,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/b663a6e225984dfca85c5d6d02794d5a to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/b663a6e225984dfca85c5d6d02794d5a 2024-11-24T16:43:39,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/3c728a12eef1458882018bbbfbc1b037 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/3c728a12eef1458882018bbbfbc1b037 2024-11-24T16:43:39,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/082f0db458ea47488992f921c32fcba4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/082f0db458ea47488992f921c32fcba4 2024-11-24T16:43:39,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e093f4a908e044e3b8645b62a6fd09f4 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/e093f4a908e044e3b8645b62a6fd09f4 2024-11-24T16:43:39,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/36f79aef759f4f1093aa9d5ce78504b5 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/36f79aef759f4f1093aa9d5ce78504b5 2024-11-24T16:43:39,659 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/recovered.edits/514.seqid, newMaxSeqId=514, maxSeqId=4 2024-11-24T16:43:39,660 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. 2024-11-24T16:43:39,660 DEBUG [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 8e3281f5174968dfe4494f3e286572d3: Waiting for close lock at 1732466617936Running coprocessor pre-close hooks at 1732466617936Disabling compacts and flushes for region at 1732466617936Disabling writes for close at 1732466617936Obtaining lock to block concurrent updates at 1732466617936Preparing flush snapshotting stores in 8e3281f5174968dfe4494f3e286572d3 at 1732466617936Finished memstore snapshotting TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732466617937 (+1 ms)Flushing stores of TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3. at 1732466617937Flushing 8e3281f5174968dfe4494f3e286572d3/A: creating writer at 1732466617937Flushing mob file 8e3281f5174968dfe4494f3e286572d3/A: appending metadata at 1732466617941 (+4 ms)Flushing mob file 8e3281f5174968dfe4494f3e286572d3/A: closing flushed file at 1732466617941Flushing 8e3281f5174968dfe4494f3e286572d3/A: appending metadata at 1732466618348 (+407 ms)Flushing 8e3281f5174968dfe4494f3e286572d3/A: closing flushed file at 1732466618348Flushing 8e3281f5174968dfe4494f3e286572d3/B: creating writer at 1732466618755 (+407 ms)Flushing 8e3281f5174968dfe4494f3e286572d3/B: appending metadata at 1732466618757 (+2 ms)Flushing 8e3281f5174968dfe4494f3e286572d3/B: closing flushed file at 1732466618757Flushing 8e3281f5174968dfe4494f3e286572d3/C: creating writer at 1732466619164 (+407 ms)Flushing 8e3281f5174968dfe4494f3e286572d3/C: appending metadata at 1732466619165 (+1 ms)Flushing 8e3281f5174968dfe4494f3e286572d3/C: closing flushed file at 1732466619165Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@754f4e81: reopening flushed file at 1732466619572 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7516e1df: reopening flushed file at 1732466619576 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@595a3d28: reopening flushed file at 1732466619579 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 8e3281f5174968dfe4494f3e286572d3 in 1646ms, sequenceid=511, compaction requested=false at 1732466619582 (+3 ms)Writing region close event to WAL at 1732466619657 (+75 ms)Running coprocessor post-close hooks at 1732466619660 (+3 ms)Closed at 1732466619660 2024-11-24T16:43:39,662 INFO [RS_CLOSE_REGION-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,663 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=8e3281f5174968dfe4494f3e286572d3, regionState=CLOSED 2024-11-24T16:43:39,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 because future has completed 2024-11-24T16:43:39,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-24T16:43:39,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 8e3281f5174968dfe4494f3e286572d3, server=e335232f7491,33607,1732466431346 in 1.8810 sec 2024-11-24T16:43:39,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-24T16:43:39,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8e3281f5174968dfe4494f3e286572d3, UNASSIGN in 1.8850 sec 2024-11-24T16:43:39,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-24T16:43:39,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8880 sec 2024-11-24T16:43:39,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732466619671"}]},"ts":"1732466619671"} 2024-11-24T16:43:39,672 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T16:43:39,672 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T16:43:39,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8960 sec 2024-11-24T16:43:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T16:43:39,924 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:39,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T16:43:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:39,926 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T16:43:39,927 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:39,929 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,931 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C, FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/recovered.edits] 2024-11-24T16:43:39,933 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/348c1e3b43f24117917210ed7e7a4e74 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/348c1e3b43f24117917210ed7e7a4e74 2024-11-24T16:43:39,933 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/7385a13cc14a4ea08101623970f71914 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/A/7385a13cc14a4ea08101623970f71914 2024-11-24T16:43:39,935 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/14027f6917c740af870dd14ca9e7c02d to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/14027f6917c740af870dd14ca9e7c02d 2024-11-24T16:43:39,936 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/a6a3820fd488402695f51fe0c817def7 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/B/a6a3820fd488402695f51fe0c817def7 2024-11-24T16:43:39,937 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0b33144898a746d1b8b1cbc8f6a0ca4b to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/0b33144898a746d1b8b1cbc8f6a0ca4b 2024-11-24T16:43:39,938 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/16f0df51ba774611881355197d066605 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/C/16f0df51ba774611881355197d066605 2024-11-24T16:43:39,940 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/recovered.edits/514.seqid to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3/recovered.edits/514.seqid 2024-11-24T16:43:39,941 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/default/TestAcidGuarantees/8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,941 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T16:43:39,941 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T16:43:39,941 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T16:43:39,943 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240578de832bb44106aeba5247cb52d48f_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240578de832bb44106aeba5247cb52d48f_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,944 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112411980183e09642baada18facc6ca8c6d_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112411980183e09642baada18facc6ca8c6d_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,945 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241de4d6e048da4a058d72d8204eb46c42_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241de4d6e048da4a058d72d8204eb46c42_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,946 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241f66d5215a1c4c22bd8558f1944d9421_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241f66d5215a1c4c22bd8558f1944d9421_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,946 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112424e12380756c409795d1ed1d5296eace_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112424e12380756c409795d1ed1d5296eace_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,947 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ad8d58f37be4328bb954fb83218992d_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ad8d58f37be4328bb954fb83218992d_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,948 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112452b5528df27242bda031058469add30c_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112452b5528df27242bda031058469add30c_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,949 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245e9f721d5d6d475bb82375c70449225d_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245e9f721d5d6d475bb82375c70449225d_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,950 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124626c08be5ee143638c5f5a66f791adf5_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124626c08be5ee143638c5f5a66f791adf5_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,950 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a3ec342b76b43bd9f59e26c77a7ca97_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a3ec342b76b43bd9f59e26c77a7ca97_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,951 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124904f3a4403164761be476e21cdf9eb24_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124904f3a4403164761be476e21cdf9eb24_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,952 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124927ab049134c45b6a81c5282a3a22edb_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124927ab049134c45b6a81c5282a3a22edb_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,953 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124add438e10d5d40069fcefc688c462e6c_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124add438e10d5d40069fcefc688c462e6c_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,954 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bf47c32ab7e64d4fb532d95e3761845c_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bf47c32ab7e64d4fb532d95e3761845c_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,955 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd18f2d15845457f913b0b0661ba1380_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cd18f2d15845457f913b0b0661ba1380_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,955 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d542b59a6c4f413b9afddf366b880ab5_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d542b59a6c4f413b9afddf366b880ab5_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,956 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e062fd48de8d40be887e398d474fe3c9_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e062fd48de8d40be887e398d474fe3c9_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,957 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e3c10c0daa9540708d07eb697d0fbbae_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e3c10c0daa9540708d07eb697d0fbbae_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,958 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f06acadaa2624f59b20ae1e7921471d9_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f06acadaa2624f59b20ae1e7921471d9_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,958 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fdf5c4b99cc140449c8ba6f2b045faf5_8e3281f5174968dfe4494f3e286572d3 to hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fdf5c4b99cc140449c8ba6f2b045faf5_8e3281f5174968dfe4494f3e286572d3 2024-11-24T16:43:39,959 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T16:43:39,961 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:39,963 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T16:43:39,964 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T16:43:39,965 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:39,965 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T16:43:39,965 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732466619965"}]},"ts":"9223372036854775807"} 2024-11-24T16:43:39,967 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T16:43:39,967 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8e3281f5174968dfe4494f3e286572d3, NAME => 'TestAcidGuarantees,,1732466584912.8e3281f5174968dfe4494f3e286572d3.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T16:43:39,967 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T16:43:39,967 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732466619967"}]},"ts":"9223372036854775807"} 2024-11-24T16:43:39,968 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T16:43:39,969 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T16:43:39,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-24T16:43:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37979 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T16:43:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T16:43:40,034 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T16:43:40,043 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 244) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 380), ProcessCount=11 (was 11), AvailableMemoryMB=5643 (was 5752) 2024-11-24T16:43:40,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T16:43:40,043 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:43:40,043 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:43:40,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:40,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:40,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T16:43:40,043 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T16:43:40,043 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=611241949, stopped=false 2024-11-24T16:43:40,044 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e335232f7491,37979,1732466430336 2024-11-24T16:43:40,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:43:40,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T16:43:40,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:43:40,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:43:40,046 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T16:43:40,046 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:43:40,046 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T16:43:40,047 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:43:40,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T16:43:40,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:40,047 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e335232f7491,33607,1732466431346' ***** 2024-11-24T16:43:40,047 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T16:43:40,048 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(959): stopping server e335232f7491,33607,1732466431346 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e335232f7491:33607. 2024-11-24T16:43:40,048 DEBUG [RS:0;e335232f7491:33607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T16:43:40,048 DEBUG [RS:0;e335232f7491:33607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T16:43:40,048 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T16:43:40,049 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T16:43:40,049 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T16:43:40,049 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:40,049 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T16:43:40,049 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T16:43:40,049 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T16:43:40,049 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T16:43:40,049 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T16:43:40,049 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-24T16:43:40,074 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/info/0900110a2ddb497fb59b5aa1ad460fbe is 95, key is TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba./info:/1732466462279/DeleteFamily/seqid=0 2024-11-24T16:43:40,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742493_1669 (size=6514) 2024-11-24T16:43:40,086 INFO [regionserver/e335232f7491:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T16:43:40,086 INFO [regionserver/e335232f7491:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T16:43:40,123 INFO [regionserver/e335232f7491:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:43:40,250 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:40,450 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:40,478 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/info/0900110a2ddb497fb59b5aa1ad460fbe 2024-11-24T16:43:40,503 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/ns/da35ae569cab4dd4b94323807837feaa is 93, key is TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba./ns:/1732466462195/DeleteFamily/seqid=0 2024-11-24T16:43:40,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742494_1670 (size=5906) 2024-11-24T16:43:40,650 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:40,789 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T16:43:40,850 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:40,906 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/ns/da35ae569cab4dd4b94323807837feaa 2024-11-24T16:43:40,924 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/rep_barrier/18d04a84e46845b899beeeeade77a940 is 102, key is TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba./rep_barrier:/1732466462195/DeleteFamily/seqid=0 2024-11-24T16:43:40,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742495_1671 (size=6025) 2024-11-24T16:43:41,050 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:41,251 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:41,328 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/rep_barrier/18d04a84e46845b899beeeeade77a940 2024-11-24T16:43:41,345 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/table/ade7c289395846b493d4b3f8d173f715 is 96, key is TestAcidGuarantees,,1732466434583.950b8a656f36ed921dc512c3b64e83ba./table:/1732466462195/DeleteFamily/seqid=0 2024-11-24T16:43:41,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742496_1672 (size=5936) 2024-11-24T16:43:41,451 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:41,651 DEBUG [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T16:43:41,748 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/table/ade7c289395846b493d4b3f8d173f715 2024-11-24T16:43:41,751 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/info/0900110a2ddb497fb59b5aa1ad460fbe as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/info/0900110a2ddb497fb59b5aa1ad460fbe 2024-11-24T16:43:41,754 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/info/0900110a2ddb497fb59b5aa1ad460fbe, entries=12, sequenceid=90, filesize=6.4 K 2024-11-24T16:43:41,755 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/ns/da35ae569cab4dd4b94323807837feaa as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/ns/da35ae569cab4dd4b94323807837feaa 2024-11-24T16:43:41,757 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/ns/da35ae569cab4dd4b94323807837feaa, entries=8, sequenceid=90, filesize=5.8 K 2024-11-24T16:43:41,758 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/rep_barrier/18d04a84e46845b899beeeeade77a940 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/rep_barrier/18d04a84e46845b899beeeeade77a940 2024-11-24T16:43:41,760 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/rep_barrier/18d04a84e46845b899beeeeade77a940, entries=6, sequenceid=90, filesize=5.9 K 2024-11-24T16:43:41,761 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/.tmp/table/ade7c289395846b493d4b3f8d173f715 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/table/ade7c289395846b493d4b3f8d173f715 2024-11-24T16:43:41,763 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/table/ade7c289395846b493d4b3f8d173f715, entries=7, sequenceid=90, filesize=5.8 K 2024-11-24T16:43:41,764 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false 2024-11-24T16:43:41,767 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-24T16:43:41,768 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T16:43:41,768 INFO [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T16:43:41,768 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732466620049Running coprocessor pre-close hooks at 1732466620049Disabling compacts and flushes for region at 1732466620049Disabling writes for close at 1732466620049Obtaining lock to block concurrent updates at 1732466620050 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732466620050Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732466620050Flushing stores of hbase:meta,,1.1588230740 at 1732466620050Flushing 1588230740/info: creating writer at 1732466620051 (+1 ms)Flushing 1588230740/info: appending metadata at 1732466620073 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732466620073Flushing 1588230740/ns: creating writer at 1732466620482 (+409 ms)Flushing 1588230740/ns: appending metadata at 1732466620502 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732466620503 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1732466620910 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732466620924 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732466620924Flushing 1588230740/table: creating writer at 1732466621331 (+407 ms)Flushing 1588230740/table: appending metadata at 1732466621345 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732466621345Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23700b86: reopening flushed file at 1732466621751 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f800bad: reopening flushed file at 1732466621754 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51eaca8: reopening flushed file at 1732466621757 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f660b2: reopening flushed file at 1732466621760 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1715ms, sequenceid=90, compaction requested=false at 1732466621764 (+4 ms)Writing region close event to WAL at 1732466621765 (+1 ms)Running coprocessor post-close hooks at 1732466621767 (+2 ms)Closed at 1732466621768 (+1 ms) 2024-11-24T16:43:41,768 DEBUG [RS_CLOSE_META-regionserver/e335232f7491:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T16:43:41,851 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(976): stopping server e335232f7491,33607,1732466431346; all regions closed. 2024-11-24T16:43:41,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741834_1010 (size=25344) 2024-11-24T16:43:41,857 DEBUG [RS:0;e335232f7491:33607 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/oldWALs 2024-11-24T16:43:41,858 INFO [RS:0;e335232f7491:33607 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C33607%2C1732466431346.meta:.meta(num 1732466434150) 2024-11-24T16:43:41,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741832_1008 (size=20051450) 2024-11-24T16:43:41,861 DEBUG [RS:0;e335232f7491:33607 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/oldWALs 2024-11-24T16:43:41,861 INFO [RS:0;e335232f7491:33607 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e335232f7491%2C33607%2C1732466431346:(num 1732466433244) 2024-11-24T16:43:41,861 DEBUG [RS:0;e335232f7491:33607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T16:43:41,861 INFO [RS:0;e335232f7491:33607 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T16:43:41,861 INFO [RS:0;e335232f7491:33607 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:43:41,861 INFO [RS:0;e335232f7491:33607 {}] hbase.ChoreService(370): Chore service for: regionserver/e335232f7491:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T16:43:41,861 INFO [RS:0;e335232f7491:33607 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:43:41,861 INFO [regionserver/e335232f7491:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:43:41,862 INFO [RS:0;e335232f7491:33607 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33607 2024-11-24T16:43:41,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e335232f7491,33607,1732466431346 2024-11-24T16:43:41,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T16:43:41,866 INFO [RS:0;e335232f7491:33607 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:43:41,869 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e335232f7491,33607,1732466431346] 2024-11-24T16:43:41,872 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e335232f7491,33607,1732466431346 already deleted, retry=false 2024-11-24T16:43:41,872 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e335232f7491,33607,1732466431346 expired; onlineServers=0 2024-11-24T16:43:41,873 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e335232f7491,37979,1732466430336' ***** 2024-11-24T16:43:41,873 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T16:43:41,873 INFO [M:0;e335232f7491:37979 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T16:43:41,873 INFO [M:0;e335232f7491:37979 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T16:43:41,873 DEBUG [M:0;e335232f7491:37979 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T16:43:41,873 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T16:43:41,873 DEBUG [M:0;e335232f7491:37979 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T16:43:41,873 DEBUG [master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466432899 {}] cleaner.HFileCleaner(306): Exit Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.small.0-1732466432899,5,FailOnTimeoutGroup] 2024-11-24T16:43:41,873 DEBUG [master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466432898 {}] cleaner.HFileCleaner(306): Exit Thread[master/e335232f7491:0:becomeActiveMaster-HFileCleaner.large.0-1732466432898,5,FailOnTimeoutGroup] 2024-11-24T16:43:41,873 INFO [M:0;e335232f7491:37979 {}] hbase.ChoreService(370): Chore service for: master/e335232f7491:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T16:43:41,873 INFO [M:0;e335232f7491:37979 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T16:43:41,874 DEBUG [M:0;e335232f7491:37979 {}] master.HMaster(1795): Stopping service threads 2024-11-24T16:43:41,874 INFO [M:0;e335232f7491:37979 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T16:43:41,874 INFO [M:0;e335232f7491:37979 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T16:43:41,874 ERROR [M:0;e335232f7491:37979 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:36643 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36643,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-24T16:43:41,874 INFO [M:0;e335232f7491:37979 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T16:43:41,874 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T16:43:41,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T16:43:41,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T16:43:41,875 DEBUG [M:0;e335232f7491:37979 {}] zookeeper.ZKUtil(347): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T16:43:41,875 WARN [M:0;e335232f7491:37979 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T16:43:41,876 INFO [M:0;e335232f7491:37979 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/.lastflushedseqids 2024-11-24T16:43:41,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742497_1673 (size=72) 2024-11-24T16:43:41,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:43:41,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33607-0x1014ad9b8770001, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:43:41,970 INFO [RS:0;e335232f7491:33607 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:43:41,970 INFO [RS:0;e335232f7491:33607 {}] regionserver.HRegionServer(1031): Exiting; stopping=e335232f7491,33607,1732466431346; zookeeper connection closed. 2024-11-24T16:43:41,970 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@204c651d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@204c651d 2024-11-24T16:43:41,971 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T16:43:42,287 INFO [M:0;e335232f7491:37979 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T16:43:42,287 INFO [M:0;e335232f7491:37979 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T16:43:42,288 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T16:43:42,288 INFO [M:0;e335232f7491:37979 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:43:42,288 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:43:42,288 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T16:43:42,288 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:43:42,288 INFO [M:0;e335232f7491:37979 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.22 KB heapSize=949.58 KB 2024-11-24T16:43:42,303 DEBUG [M:0;e335232f7491:37979 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a3a2229416a14251a6c1aef05502866e is 82, key is hbase:meta,,1/info:regioninfo/1732466434241/Put/seqid=0 2024-11-24T16:43:42,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742498_1674 (size=5672) 2024-11-24T16:43:42,706 INFO [M:0;e335232f7491:37979 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2154 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a3a2229416a14251a6c1aef05502866e 2024-11-24T16:43:42,725 DEBUG [M:0;e335232f7491:37979 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcdd580f158f4439947324d1d6576823 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1732466587988/Put/seqid=0 2024-11-24T16:43:42,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742499_1675 (size=45415) 2024-11-24T16:43:43,129 INFO [M:0;e335232f7491:37979 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=774.66 KB at sequenceid=2154 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcdd580f158f4439947324d1d6576823 2024-11-24T16:43:43,132 INFO [M:0;e335232f7491:37979 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dcdd580f158f4439947324d1d6576823 2024-11-24T16:43:43,146 DEBUG [M:0;e335232f7491:37979 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79aad2fce00a498ea3180364ab73d803 is 69, key is e335232f7491,33607,1732466431346/rs:state/1732466433010/Put/seqid=0 2024-11-24T16:43:43,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073742500_1676 (size=5156) 2024-11-24T16:43:43,550 INFO [M:0;e335232f7491:37979 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2154 (bloomFilter=true), to=hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79aad2fce00a498ea3180364ab73d803 2024-11-24T16:43:43,553 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a3a2229416a14251a6c1aef05502866e as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a3a2229416a14251a6c1aef05502866e 2024-11-24T16:43:43,555 INFO [M:0;e335232f7491:37979 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a3a2229416a14251a6c1aef05502866e, entries=8, sequenceid=2154, filesize=5.5 K 2024-11-24T16:43:43,556 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dcdd580f158f4439947324d1d6576823 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dcdd580f158f4439947324d1d6576823 2024-11-24T16:43:43,558 INFO [M:0;e335232f7491:37979 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dcdd580f158f4439947324d1d6576823 2024-11-24T16:43:43,558 INFO [M:0;e335232f7491:37979 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dcdd580f158f4439947324d1d6576823, entries=170, sequenceid=2154, filesize=44.4 K 2024-11-24T16:43:43,559 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/79aad2fce00a498ea3180364ab73d803 as hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/79aad2fce00a498ea3180364ab73d803 2024-11-24T16:43:43,561 INFO [M:0;e335232f7491:37979 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36643/user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/79aad2fce00a498ea3180364ab73d803, entries=1, sequenceid=2154, filesize=5.0 K 2024-11-24T16:43:43,561 INFO [M:0;e335232f7491:37979 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.22 KB/793823, heapSize ~949.28 KB/972064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1273ms, sequenceid=2154, compaction requested=false 2024-11-24T16:43:43,563 INFO [M:0;e335232f7491:37979 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T16:43:43,563 DEBUG [M:0;e335232f7491:37979 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732466622287Disabling compacts and flushes for region at 1732466622287Disabling writes for close at 1732466622288 (+1 ms)Obtaining lock to block concurrent updates at 1732466622288Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732466622288Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=793823, getHeapSize=972304, getOffHeapSize=0, getCellsCount=2349 at 1732466622288Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732466622288Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732466622289 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732466622303 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732466622303Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732466622709 (+406 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732466622725 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732466622725Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732466623132 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732466623146 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732466623146Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ff406b6: reopening flushed file at 1732466623552 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b46d90a: reopening flushed file at 1732466623555 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47ded1fc: reopening flushed file at 1732466623558 (+3 ms)Finished flush of dataSize ~775.22 KB/793823, heapSize ~949.28 KB/972064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1273ms, sequenceid=2154, compaction requested=false at 1732466623561 (+3 ms)Writing region close event to WAL at 1732466623563 (+2 ms)Closed at 1732466623563 2024-11-24T16:43:43,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34923 is added to blk_1073741830_1006 (size=935032) 2024-11-24T16:43:43,565 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/063db4f3-bc46-e3be-64aa-3099ed76a446/MasterData/WALs/e335232f7491,37979,1732466430336/e335232f7491%2C37979%2C1732466430336.1732466432245 not finished, retry = 0 2024-11-24T16:43:43,666 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T16:43:43,667 INFO [M:0;e335232f7491:37979 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T16:43:43,667 INFO [M:0;e335232f7491:37979 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37979 2024-11-24T16:43:43,667 INFO [M:0;e335232f7491:37979 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T16:43:43,769 INFO [M:0;e335232f7491:37979 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T16:43:43,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:43:43,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37979-0x1014ad9b8770000, quorum=127.0.0.1:58212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T16:43:43,774 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T16:43:43,776 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T16:43:43,776 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T16:43:43,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T16:43:43,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/hadoop.log.dir/,STOPPED} 2024-11-24T16:43:43,779 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T16:43:43,779 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T16:43:43,779 WARN [BP-538122245-172.17.0.2-1732466426978 heartbeating to localhost/127.0.0.1:36643 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T16:43:43,779 WARN [BP-538122245-172.17.0.2-1732466426978 heartbeating to localhost/127.0.0.1:36643 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-538122245-172.17.0.2-1732466426978 (Datanode Uuid d7540ee3-cd6e-4ff0-9f26-dcf559838e09) service to localhost/127.0.0.1:36643 2024-11-24T16:43:43,782 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/data/data1/current/BP-538122245-172.17.0.2-1732466426978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T16:43:43,782 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/cluster_8849cafb-1aa9-af1e-ad28-3f30a8bea04c/data/data2/current/BP-538122245-172.17.0.2-1732466426978 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T16:43:43,783 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T16:43:43,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T16:43:43,791 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T16:43:43,791 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T16:43:43,791 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T16:43:43,791 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/52fb54c9-e0b1-1f92-2cfb-11091f81b9e2/hadoop.log.dir/,STOPPED} 2024-11-24T16:43:43,808 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T16:43:43,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down